001package io.ebean.migration.runner;
002
003import io.ebean.ddlrunner.ScriptTransform;
004import io.ebean.migration.JdbcMigration;
005import io.ebean.migration.MigrationConfig;
006import io.ebean.migration.MigrationException;
007import io.ebean.migration.MigrationVersion;
008import org.slf4j.Logger;
009import org.slf4j.LoggerFactory;
010
011import java.io.IOException;
012import java.net.URL;
013import java.sql.Connection;
014import java.sql.DatabaseMetaData;
015import java.sql.ResultSet;
016import java.sql.SQLException;
017import java.sql.Timestamp;
018import java.util.ArrayList;
019import java.util.Enumeration;
020import java.util.LinkedHashMap;
021import java.util.List;
022import java.util.Set;
023
024import static io.ebean.migration.MigrationVersion.BOOTINIT_TYPE;
025import static io.ebean.migration.MigrationVersion.VERSION_TYPE;
026
027/**
028 * Manages the migration table.
029 */
030public class MigrationTable {
031
032  private static final Logger logger = LoggerFactory.getLogger("io.ebean.DDL");
033
034  private static final String INIT_VER_0 = "0";
035
036  private final Connection connection;
037  private final boolean checkState;
038  private final MigrationPlatform platform;
039  private final MigrationScriptRunner scriptRunner;
040  private final String catalog;
041  private final String schema;
042  private final String table;
043  private final String sqlTable;
044  private final String envUserName;
045  private final String platformName;
046
047  private final Timestamp runOn = new Timestamp(System.currentTimeMillis());
048
049  private final ScriptTransform scriptTransform;
050
051  private final String insertSql;
052  private final String updateSql;
053  private final String updateChecksumSql;
054
055  private final LinkedHashMap<String, MigrationMetaRow> migrations;
056  private final boolean skipChecksum;
057  private final boolean skipMigrationRun;
058
059  private final Set<String> patchInsertVersions;
060  private final Set<String> patchResetChecksumVersions;
061  private final boolean allowErrorInRepeatable;
062
063  private final MigrationVersion minVersion;
064  private final String minVersionFailMessage;
065
066  private MigrationVersion currentVersion;
067  private MigrationMetaRow lastMigration;
068  private LocalMigrationResource priorVersion;
069
070  private final List<LocalMigrationResource> checkMigrations = new ArrayList<>();
071
072  /**
073   * Version of a dbinit script. When set this means all migration version less than this are ignored.
074   */
075  private MigrationVersion dbInitVersion;
076
077  /**
078   * Construct with server, configuration and jdbc connection (DB admin user).
079   */
080  public MigrationTable(MigrationConfig config, Connection connection, boolean checkState, MigrationPlatform platform) {
081    this.platform = platform;
082    this.connection = connection;
083    this.scriptRunner = new MigrationScriptRunner(connection, platform);
084    this.checkState = checkState;
085    this.migrations = new LinkedHashMap<>();
086    this.catalog = null;
087    this.allowErrorInRepeatable = config.isAllowErrorInRepeatable();
088    this.patchResetChecksumVersions = config.getPatchResetChecksumOn();
089    this.patchInsertVersions = config.getPatchInsertOn();
090    this.minVersion = initMinVersion(config.getMinVersion());
091    this.minVersionFailMessage = config.getMinVersionFailMessage();
092    this.skipMigrationRun = config.isSkipMigrationRun();
093    this.skipChecksum = config.isSkipChecksum();
094    this.schema = config.getDbSchema();
095    this.table = config.getMetaTable();
096    this.platformName = config.getPlatformName();
097    this.sqlTable = initSqlTable();
098    this.insertSql = MigrationMetaRow.insertSql(sqlTable);
099    this.updateSql = MigrationMetaRow.updateSql(sqlTable);
100    this.updateChecksumSql = MigrationMetaRow.updateChecksumSql(sqlTable);
101    this.scriptTransform = createScriptTransform(config);
102    this.envUserName = System.getProperty("user.name");
103  }
104
105  private MigrationVersion initMinVersion(String minVersion) {
106    return (minVersion == null || minVersion.isEmpty()) ? null : MigrationVersion.parse(minVersion);
107  }
108
109  private String initSqlTable() {
110    if (schema != null) {
111      return schema + "." + table;
112    } else {
113      return table;
114    }
115  }
116
117  private String sqlPrimaryKey() {
118    return "pk_" + table;
119  }
120
121  /**
122   * Return the number of migrations in the DB migration table.
123   */
124  public int size() {
125    return migrations.size();
126  }
127
128  /**
129   * Returns the versions that are already applied.
130   */
131  public Set<String> getVersions() {
132    return migrations.keySet();
133  }
134
135  /**
136   * Create the ScriptTransform for placeholder key/value replacement.
137   */
138  private ScriptTransform createScriptTransform(MigrationConfig config) {
139    return ScriptTransform.build(config.getRunPlaceholders(), config.getRunPlaceholderMap());
140  }
141
142  /**
143   * Create the table is it does not exist.
144   * <p>
145   * Also holds DB lock on migration table and loads existing migrations.
146   * </p>
147   */
148  public void createIfNeededAndLock() throws SQLException, IOException {
149    if (!tableExists()) {
150      createTable();
151    }
152    obtainLockWithWait();
153    readExistingMigrations();
154  }
155
156  /**
157   * Obtain lock with wait, note that other nodes can insert and commit
158   * into the migration table during the wait so this query result won't
159   * contain all the executed migrations in that case.
160   */
161  private void obtainLockWithWait() throws SQLException {
162    platform.lockMigrationTable(sqlTable, connection);
163  }
164
165  /**
166   * Read the migration table with details on what migrations have run.
167   * This must execute after we have completed the wait for the lock on
168   * the migration table such that it reads any migrations that have
169   * executed during the wait for the lock.
170   */
171  private void readExistingMigrations() throws SQLException {
172    for (MigrationMetaRow metaRow : platform.readExistingMigrations(sqlTable, connection)) {
173      addMigration(metaRow.getVersion(), metaRow);
174    }
175  }
176
177  private void createTable() throws IOException, SQLException {
178    scriptRunner.runScript(createTableDdl(), "create migration table");
179    createInitMetaRow().executeInsert(connection, insertSql);
180  }
181
182  /**
183   * Return the create table script.
184   */
185  String createTableDdl() throws IOException {
186    String script = ScriptTransform.replace("${table}", sqlTable, getCreateTableScript());
187    return ScriptTransform.replace("${pk_table}", sqlPrimaryKey(), script);
188  }
189
190  /**
191   * Return the create table script.
192   */
193  private String getCreateTableScript() throws IOException {
194    // supply a script to override the default table create script
195    String script = readResource("migration-support/create-table.sql");
196    if (script == null && platformName != null && !platformName.isEmpty()) {
197      // look for platform specific create table
198      script = readResource("migration-support/" + platformName + "-create-table.sql");
199    }
200    if (script == null) {
201      // no, just use the default script
202      script = readResource("migration-support/default-create-table.sql");
203    }
204    return script;
205  }
206
207  private String readResource(String location) throws IOException {
208    Enumeration<URL> resources = getClassLoader().getResources(location);
209    if (resources.hasMoreElements()) {
210      URL url = resources.nextElement();
211      return IOUtils.readUtf8(url);
212    }
213    return null;
214  }
215
216  private ClassLoader getClassLoader() {
217    return Thread.currentThread().getContextClassLoader();
218  }
219
220  /**
221   * Return true if the table exists.
222   */
223  private boolean tableExists() throws SQLException {
224    String migTable = table;
225    DatabaseMetaData metaData = connection.getMetaData();
226    if (metaData.storesUpperCaseIdentifiers()) {
227      migTable = migTable.toUpperCase();
228    }
229    String checkCatalog = (catalog != null) ? catalog : connection.getCatalog();
230    String checkSchema = (schema != null) ? schema : connection.getSchema();
231    try (ResultSet tables = metaData.getTables(checkCatalog, checkSchema, migTable, null)) {
232      return tables.next();
233    }
234  }
235
236  /**
237   * Return true if the migration ran successfully and false if the migration failed.
238   */
239  private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationResource prior) throws SQLException {
240    if (prior != null && !localVersion.isRepeatable()) {
241      if (!migrationExists(prior)) {
242        logger.error("Migration {} requires prior migration {} which has not been run", localVersion.getVersion(), prior.getVersion());
243        return false;
244      }
245    }
246
247    MigrationMetaRow existing = migrations.get(localVersion.key());
248    if (!runMigration(localVersion, existing)) {
249      return false;
250    }
251
252    // migration was run successfully ...
253    priorVersion = localVersion;
254    return true;
255  }
256
257  /**
258   * Run the migration script.
259   *
260   * @param local    The local migration resource
261   * @param existing The information for this migration existing in the table
262   * @return True if the migrations should continue
263   */
264  private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
265    String script = null;
266    int checksum;
267    if (local instanceof LocalDdlMigrationResource) {
268      script = convertScript(local.getContent());
269      checksum = Checksum.calculate(script);
270    } else {
271      checksum = ((LocalJdbcMigrationResource) local).getChecksum();
272    }
273
274    if (existing == null && patchInsertMigration(local, checksum)) {
275      return true;
276    }
277    if (existing != null && skipMigration(checksum, local, existing)) {
278      return true;
279    }
280    executeMigration(local, script, checksum, existing);
281    return true;
282  }
283
284  /**
285   * Return true if we 'patch history' inserting a DB migration without running it.
286   */
287  private boolean patchInsertMigration(LocalMigrationResource local, int checksum) throws SQLException {
288    if (patchInsertVersions != null && patchInsertVersions.contains(local.key())) {
289      logger.info("patch migration - insert into history {}", local.getLocation());
290      if (!checkState) {
291        insertIntoHistory(local, checksum, 0);
292      }
293      return true;
294    }
295    return false;
296  }
297
298  /**
299   * Return true if the migration should be skipped.
300   */
301  boolean skipMigration(int checksum, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
302    boolean matchChecksum = (existing.getChecksum() == checksum);
303    if (matchChecksum) {
304      logger.trace("... skip unchanged migration {}", local.getLocation());
305      return true;
306
307    } else if (patchResetChecksum(existing, checksum)) {
308      logger.info("patch migration - reset checksum on {}", local.getLocation());
309      return true;
310
311    } else if (local.isRepeatable() || skipChecksum) {
312      // re-run the migration
313      return false;
314    } else {
315      throw new MigrationException("Checksum mismatch on migration " + local.getLocation());
316    }
317  }
318
319  /**
320   * Return true if the checksum is reset on the existing migration.
321   */
322  private boolean patchResetChecksum(MigrationMetaRow existing, int newChecksum) throws SQLException {
323    if (isResetOnVersion(existing.getVersion())) {
324      if (!checkState) {
325        existing.resetChecksum(newChecksum, connection, updateChecksumSql);
326      }
327      return true;
328    } else {
329      return false;
330    }
331  }
332
333  private boolean isResetOnVersion(String version) {
334    return patchResetChecksumVersions != null && patchResetChecksumVersions.contains(version);
335  }
336
337  /**
338   * Run a migration script as new migration or update on existing repeatable migration.
339   */
340  private void executeMigration(LocalMigrationResource local, String script, int checksum, MigrationMetaRow existing) throws SQLException {
341    if (checkState) {
342      checkMigrations.add(local);
343      // simulate the migration being run such that following migrations also match
344      addMigration(local.key(), createMetaRow(local, checksum, 1));
345      return;
346    }
347
348    long exeMillis = 0;
349    try {
350      if (skipMigrationRun) {
351        logger.debug("skip migration {}", local.getLocation());
352      } else {
353        exeMillis = executeMigration(local, script);
354      }
355      if (existing != null) {
356        existing.rerun(checksum, exeMillis, envUserName, runOn);
357        existing.executeUpdate(connection, updateSql);
358      } else {
359        insertIntoHistory(local, checksum, exeMillis);
360      }
361    } catch (SQLException e) {
362      if (allowErrorInRepeatable && local.isRepeatableLast()) {
363        // log the exception and continue on repeatable migration
364        logger.error("Continue migration with error executing repeatable migration " + local.getVersion(), e);
365      } else {
366        throw e;
367      }
368    }
369  }
370
371  private long executeMigration(LocalMigrationResource local, String script) throws SQLException {
372    long start = System.currentTimeMillis();
373    if (local instanceof LocalDdlMigrationResource) {
374      logger.debug("run migration {}", local.getLocation());
375      scriptRunner.runScript(script, "run migration version: " + local.getVersion());
376    } else {
377      JdbcMigration migration = ((LocalJdbcMigrationResource) local).getMigration();
378      logger.info("Executing jdbc migration version: {} - {}", local.getVersion(), migration);
379      migration.migrate(connection);
380    }
381    return System.currentTimeMillis() - start;
382  }
383
384  private void insertIntoHistory(LocalMigrationResource local, int checksum, long exeMillis) throws SQLException {
385    MigrationMetaRow metaRow = createMetaRow(local, checksum, exeMillis);
386    metaRow.executeInsert(connection, insertSql);
387    addMigration(local.key(), metaRow);
388  }
389
390  private MigrationMetaRow createInitMetaRow() {
391    return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", 0, envUserName, runOn, 0);
392  }
393
394  /**
395   * Create the MigrationMetaRow for this migration.
396   */
397  private MigrationMetaRow createMetaRow(LocalMigrationResource migration, int checksum, long exeMillis) {
398
399    int nextId = 1;
400    if (lastMigration != null) {
401      nextId = lastMigration.getId() + 1;
402    }
403
404    String type = migration.getType();
405    String runVersion = migration.key();
406    String comment = migration.getComment();
407
408    return new MigrationMetaRow(nextId, type, runVersion, comment, checksum, envUserName, runOn, exeMillis);
409  }
410
411  /**
412   * Return true if the migration exists.
413   */
414  private boolean migrationExists(LocalMigrationResource priorVersion) {
415    return migrations.containsKey(priorVersion.key());
416  }
417
418  /**
419   * Apply the placeholder key/value replacement on the script.
420   */
421  private String convertScript(String script) {
422    return scriptTransform.transform(script);
423  }
424
425  /**
426   * Register the successfully executed migration (to allow dependant scripts to run).
427   */
428  private void addMigration(String key, MigrationMetaRow metaRow) {
429    if (INIT_VER_0.equals(key)) {
430      // ignore the version 0 <init> row
431      return;
432    }
433    lastMigration = metaRow;
434    if (metaRow.getVersion() == null) {
435      throw new IllegalStateException("No runVersion in db migration table row? " + metaRow);
436    }
437
438    migrations.put(key, metaRow);
439    if (VERSION_TYPE.equals(metaRow.getType()) || BOOTINIT_TYPE.equals(metaRow.getType())) {
440      MigrationVersion rowVersion = MigrationVersion.parse(metaRow.getVersion());
441      if (currentVersion == null || rowVersion.compareTo(currentVersion) > 0) {
442        currentVersion = rowVersion;
443      }
444      if (BOOTINIT_TYPE.equals(metaRow.getType())) {
445        dbInitVersion = rowVersion;
446      }
447    }
448  }
449
450  /**
451   * Return true if there are no migrations.
452   */
453  public boolean isEmpty() {
454    return migrations.isEmpty();
455  }
456
457  /**
458   * Run all the migrations in order as needed.
459   *
460   * @return the migrations that have been run (collected if checkstate is true).
461   */
462  public List<LocalMigrationResource> runAll(List<LocalMigrationResource> localVersions) throws SQLException {
463
464    checkMinVersion();
465    for (LocalMigrationResource localVersion : localVersions) {
466      if (!localVersion.isRepeatable() && dbInitVersion != null && dbInitVersion.compareTo(localVersion.getVersion()) >= 0) {
467        logger.debug("migration skipped by dbInitVersion {}", dbInitVersion);
468      } else if (!shouldRun(localVersion, priorVersion)) {
469        break;
470      }
471    }
472    return checkMigrations;
473  }
474
475  private void checkMinVersion() {
476    if (minVersion != null && currentVersion != null && currentVersion.compareTo(minVersion) < 0) {
477      StringBuilder sb = new StringBuilder();
478      if (minVersionFailMessage != null && !minVersionFailMessage.isEmpty()) {
479        sb.append(minVersionFailMessage).append(' ');
480      }
481      sb.append("MigrationVersion mismatch: v").append(currentVersion).append(" < v").append(minVersion);
482      throw new MigrationException(sb.toString());
483    }
484  }
485
486  /**
487   * Run using an init migration.
488   *
489   * @return the migrations that have been run (collected if checkstate is true).
490   */
491  public List<LocalMigrationResource> runInit(LocalMigrationResource initVersion, List<LocalMigrationResource> localVersions) throws SQLException {
492
493    runRepeatableInit(localVersions);
494
495    initVersion.setInitType();
496    if (!shouldRun(initVersion, null)) {
497      throw new IllegalStateException("Expected to run init migration but it didn't?");
498    }
499
500    // run any migrations greater that the init migration
501    for (LocalMigrationResource localVersion : localVersions) {
502      if (localVersion.compareTo(initVersion) > 0 && !shouldRun(localVersion, priorVersion)) {
503        break;
504      }
505    }
506    return checkMigrations;
507  }
508
509  private void runRepeatableInit(List<LocalMigrationResource> localVersions) throws SQLException {
510    for (LocalMigrationResource localVersion : localVersions) {
511      if (!localVersion.isRepeatableInit() || !shouldRun(localVersion, priorVersion)) {
512        break;
513      }
514    }
515  }
516
517  /**
518   * Run non transactional statements (if any) after migration commit.
519   * <p>
520   * These run with auto commit true and run AFTER the migration commit and
521   * as such the migration isn't truely atomic - the migration can run and
522   * complete and the non-transactional statements fail.
523   */
524  public void runNonTransactional() {
525    scriptRunner.runNonTransactional();
526  }
527}