001package io.ebean.migration.runner;
002
003import io.ebean.migration.JdbcMigration;
004import io.ebean.migration.MigrationConfig;
005import io.ebean.migration.MigrationException;
006import io.ebean.migration.MigrationVersion;
007import io.ebean.migration.util.IOUtils;
008import org.slf4j.Logger;
009import org.slf4j.LoggerFactory;
010
011import java.io.IOException;
012import java.net.URL;
013import java.sql.Connection;
014import java.sql.DatabaseMetaData;
015import java.sql.ResultSet;
016import java.sql.SQLException;
017import java.sql.Timestamp;
018import java.util.ArrayList;
019import java.util.Enumeration;
020import java.util.LinkedHashMap;
021import java.util.List;
022import java.util.Set;
023
024import static io.ebean.migration.MigrationVersion.BOOTINIT_TYPE;
025import static io.ebean.migration.MigrationVersion.VERSION_TYPE;
026
027/**
028 * Manages the migration table.
029 */
030public class MigrationTable {
031
032  private static final Logger logger = LoggerFactory.getLogger("io.ebean.DDL");
033
034  private static final String INIT_VER_0 = "0";
035
036  private final Connection connection;
037  private final boolean checkState;
038  private final MigrationPlatform platform;
039  private final MigrationScriptRunner scriptRunner;
040  private final String catalog;
041  private final String schema;
042  private final String table;
043  private final String sqlTable;
044  private final String envUserName;
045  private final String platformName;
046
047  private final Timestamp runOn = new Timestamp(System.currentTimeMillis());
048
049  private final ScriptTransform scriptTransform;
050
051  private final String insertSql;
052  private final String updateSql;
053  private final String updateChecksumSql;
054
055  private final LinkedHashMap<String, MigrationMetaRow> migrations;
056  private final boolean skipChecksum;
057
058  private final Set<String> patchInsertVersions;
059  private final Set<String> patchResetChecksumVersions;
060  private final boolean allowErrorInRepeatable;
061
062  private final MigrationVersion minVersion;
063  private final String minVersionFailMessage;
064
065  private MigrationVersion currentVersion;
066  private MigrationMetaRow lastMigration;
067  private LocalMigrationResource priorVersion;
068
069  private final List<LocalMigrationResource> checkMigrations = new ArrayList<>();
070
071  /**
072   * Version of a dbinit script. When set this means all migration version less than this are ignored.
073   */
074  private MigrationVersion dbInitVersion;
075
076  /**
077   * Construct with server, configuration and jdbc connection (DB admin user).
078   */
079  public MigrationTable(MigrationConfig config, Connection connection, boolean checkState, MigrationPlatform platform) {
080    this.platform = platform;
081    this.connection = connection;
082    this.scriptRunner = new MigrationScriptRunner(connection, platform);
083    this.checkState = checkState;
084    this.migrations = new LinkedHashMap<>();
085    this.catalog = null;
086    this.allowErrorInRepeatable = config.isAllowErrorInRepeatable();
087    this.patchResetChecksumVersions = config.getPatchResetChecksumOn();
088    this.patchInsertVersions = config.getPatchInsertOn();
089    this.minVersion = initMinVersion(config.getMinVersion());
090    this.minVersionFailMessage = config.getMinVersionFailMessage();
091    this.skipChecksum = config.isSkipChecksum();
092    this.schema = config.getDbSchema();
093    this.table = config.getMetaTable();
094    this.platformName = config.getPlatformName();
095    this.sqlTable = initSqlTable();
096    this.insertSql = MigrationMetaRow.insertSql(sqlTable);
097    this.updateSql = MigrationMetaRow.updateSql(sqlTable);
098    this.updateChecksumSql = MigrationMetaRow.updateChecksumSql(sqlTable);
099    this.scriptTransform = createScriptTransform(config);
100    this.envUserName = System.getProperty("user.name");
101  }
102
103  private MigrationVersion initMinVersion(String minVersion) {
104    return (minVersion == null || minVersion.isEmpty()) ? null : MigrationVersion.parse(minVersion);
105  }
106
107  private String initSqlTable() {
108    if (schema != null) {
109      return schema + "." + table;
110    } else {
111      return table;
112    }
113  }
114
115  private String sqlPrimaryKey() {
116    return "pk_" + table;
117  }
118
119  /**
120   * Return the number of migrations in the DB migration table.
121   */
122  public int size() {
123    return migrations.size();
124  }
125
126  /**
127   * Returns the versions that are already applied.
128   */
129  public Set<String> getVersions() {
130    return migrations.keySet();
131  }
132
133  /**
134   * Create the ScriptTransform for placeholder key/value replacement.
135   */
136  private ScriptTransform createScriptTransform(MigrationConfig config) {
137    return ScriptTransform.build(config.getRunPlaceholders(), config.getRunPlaceholderMap());
138  }
139
140  /**
141   * Create the table is it does not exist.
142   * <p>
143   * Also holds DB lock on migration table and loads existing migrations.
144   * </p>
145   */
146  public void createIfNeededAndLock() throws SQLException, IOException {
147    if (!tableExists()) {
148      createTable();
149    }
150    obtainLockWithWait();
151    readExistingMigrations();
152  }
153
154  /**
155   * Obtain lock with wait, note that other nodes can insert and commit
156   * into the migration table during the wait so this query result won't
157   * contain all the executed migrations in that case.
158   */
159  private void obtainLockWithWait() throws SQLException {
160    platform.lockMigrationTable(sqlTable, connection);
161  }
162
163  /**
164   * Read the migration table with details on what migrations have run.
165   * This must execute after we have completed the wait for the lock on
166   * the migration table such that it reads any migrations that have
167   * executed during the wait for the lock.
168   */
169  private void readExistingMigrations() throws SQLException {
170    for (MigrationMetaRow metaRow : platform.readExistingMigrations(sqlTable, connection)) {
171      addMigration(metaRow.getVersion(), metaRow);
172    }
173  }
174
175  private void createTable() throws IOException, SQLException {
176    scriptRunner.runScript(createTableDdl(), "create migration table");
177    createInitMetaRow().executeInsert(connection, insertSql);
178  }
179
180  /**
181   * Return the create table script.
182   */
183  String createTableDdl() throws IOException {
184    String script = ScriptTransform.replace("${table}", sqlTable, getCreateTableScript());
185    return ScriptTransform.replace("${pk_table}", sqlPrimaryKey(), script);
186  }
187
188  /**
189   * Return the create table script.
190   */
191  private String getCreateTableScript() throws IOException {
192    // supply a script to override the default table create script
193    String script = readResource("migration-support/create-table.sql");
194    if (script == null && platformName != null && !platformName.isEmpty()) {
195      // look for platform specific create table
196      script = readResource("migration-support/" + platformName + "-create-table.sql");
197    }
198    if (script == null) {
199      // no, just use the default script
200      script = readResource("migration-support/default-create-table.sql");
201    }
202    return script;
203  }
204
205  private String readResource(String location) throws IOException {
206    Enumeration<URL> resources = getClassLoader().getResources(location);
207    if (resources.hasMoreElements()) {
208      URL url = resources.nextElement();
209      return IOUtils.readUtf8(url);
210    }
211    return null;
212  }
213
214  private ClassLoader getClassLoader() {
215    return Thread.currentThread().getContextClassLoader();
216  }
217
218  /**
219   * Return true if the table exists.
220   */
221  private boolean tableExists() throws SQLException {
222    String migTable = table;
223    DatabaseMetaData metaData = connection.getMetaData();
224    if (metaData.storesUpperCaseIdentifiers()) {
225      migTable = migTable.toUpperCase();
226    }
227    String checkCatalog = (catalog != null) ? catalog : connection.getCatalog();
228    String checkSchema = (schema != null) ? schema : connection.getSchema();
229    try (ResultSet tables = metaData.getTables(checkCatalog, checkSchema, migTable, null)) {
230      return tables.next();
231    }
232  }
233
234  /**
235   * Return true if the migration ran successfully and false if the migration failed.
236   */
237  private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationResource prior) throws SQLException {
238    if (prior != null && !localVersion.isRepeatable()) {
239      if (!migrationExists(prior)) {
240        logger.error("Migration {} requires prior migration {} which has not been run", localVersion.getVersion(), prior.getVersion());
241        return false;
242      }
243    }
244
245    MigrationMetaRow existing = migrations.get(localVersion.key());
246    if (!runMigration(localVersion, existing)) {
247      return false;
248    }
249
250    // migration was run successfully ...
251    priorVersion = localVersion;
252    return true;
253  }
254
255  /**
256   * Run the migration script.
257   *
258   * @param local    The local migration resource
259   * @param existing The information for this migration existing in the table
260   * @return True if the migrations should continue
261   */
262  private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
263    String script = null;
264    int checksum;
265    if (local instanceof LocalDdlMigrationResource) {
266      script = convertScript(local.getContent());
267      checksum = Checksum.calculate(script);
268    } else {
269      checksum = ((LocalJdbcMigrationResource) local).getChecksum();
270    }
271
272    if (existing == null && patchInsertMigration(local, checksum)) {
273      return true;
274    }
275    if (existing != null && skipMigration(checksum, local, existing)) {
276      return true;
277    }
278    executeMigration(local, script, checksum, existing);
279    return true;
280  }
281
282  /**
283   * Return true if we 'patch history' inserting a DB migration without running it.
284   */
285  private boolean patchInsertMigration(LocalMigrationResource local, int checksum) throws SQLException {
286    if (patchInsertVersions != null && patchInsertVersions.contains(local.key())) {
287      logger.info("patch migration - insert into history {}", local.getLocation());
288      if (!checkState) {
289        insertIntoHistory(local, checksum, 0);
290      }
291      return true;
292    }
293    return false;
294  }
295
296  /**
297   * Return true if the migration should be skipped.
298   */
299  boolean skipMigration(int checksum, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
300    boolean matchChecksum = (existing.getChecksum() == checksum);
301    if (matchChecksum) {
302      logger.trace("... skip unchanged migration {}", local.getLocation());
303      return true;
304
305    } else if (patchResetChecksum(existing, checksum)) {
306      logger.info("patch migration - reset checksum on {}", local.getLocation());
307      return true;
308
309    } else if (local.isRepeatable() || skipChecksum) {
310      // re-run the migration
311      return false;
312    } else {
313      throw new MigrationException("Checksum mismatch on migration " + local.getLocation());
314    }
315  }
316
317  /**
318   * Return true if the checksum is reset on the existing migration.
319   */
320  private boolean patchResetChecksum(MigrationMetaRow existing, int newChecksum) throws SQLException {
321    if (isResetOnVersion(existing.getVersion())) {
322      if (!checkState) {
323        existing.resetChecksum(newChecksum, connection, updateChecksumSql);
324      }
325      return true;
326    } else {
327      return false;
328    }
329  }
330
331  private boolean isResetOnVersion(String version) {
332    return patchResetChecksumVersions != null && patchResetChecksumVersions.contains(version);
333  }
334
335  /**
336   * Run a migration script as new migration or update on existing repeatable migration.
337   */
338  private void executeMigration(LocalMigrationResource local, String script, int checksum, MigrationMetaRow existing) throws SQLException {
339    if (checkState) {
340      checkMigrations.add(local);
341      // simulate the migration being run such that following migrations also match
342      addMigration(local.key(), createMetaRow(local, checksum, 1));
343      return;
344    }
345
346    logger.debug("run migration {}", local.getLocation());
347
348    long start = System.currentTimeMillis();
349    try {
350      if (local instanceof LocalDdlMigrationResource) {
351        scriptRunner.runScript(script, "run migration version: " + local.getVersion());
352      } else {
353        JdbcMigration migration = ((LocalJdbcMigrationResource) local).getMigration();
354        logger.info("Executing jdbc migration version: {} - {}", local.getVersion(), migration);
355        migration.migrate(connection);
356      }
357      long exeMillis = System.currentTimeMillis() - start;
358
359      if (existing != null) {
360        existing.rerun(checksum, exeMillis, envUserName, runOn);
361        existing.executeUpdate(connection, updateSql);
362
363      } else {
364        insertIntoHistory(local, checksum, exeMillis);
365      }
366    } catch (SQLException e) {
367      if (allowErrorInRepeatable && local.isRepeatableLast()) {
368        // log the exception and continue on repeatable migration
369        logger.error("Continue migration with error executing repeatable migration " + local.getVersion(), e);
370      } else {
371        throw e;
372      }
373    }
374  }
375
376  private void insertIntoHistory(LocalMigrationResource local, int checksum, long exeMillis) throws SQLException {
377    MigrationMetaRow metaRow = createMetaRow(local, checksum, exeMillis);
378    metaRow.executeInsert(connection, insertSql);
379    addMigration(local.key(), metaRow);
380  }
381
382  private MigrationMetaRow createInitMetaRow() {
383    return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", 0, envUserName, runOn, 0);
384  }
385
386  /**
387   * Create the MigrationMetaRow for this migration.
388   */
389  private MigrationMetaRow createMetaRow(LocalMigrationResource migration, int checksum, long exeMillis) {
390
391    int nextId = 1;
392    if (lastMigration != null) {
393      nextId = lastMigration.getId() + 1;
394    }
395
396    String type = migration.getType();
397    String runVersion = migration.key();
398    String comment = migration.getComment();
399
400    return new MigrationMetaRow(nextId, type, runVersion, comment, checksum, envUserName, runOn, exeMillis);
401  }
402
403  /**
404   * Return true if the migration exists.
405   */
406  private boolean migrationExists(LocalMigrationResource priorVersion) {
407    return migrations.containsKey(priorVersion.key());
408  }
409
410  /**
411   * Apply the placeholder key/value replacement on the script.
412   */
413  private String convertScript(String script) {
414    return scriptTransform.transform(script);
415  }
416
417  /**
418   * Register the successfully executed migration (to allow dependant scripts to run).
419   */
420  private void addMigration(String key, MigrationMetaRow metaRow) {
421    if (INIT_VER_0.equals(key)) {
422      // ignore the version 0 <init> row
423      return;
424    }
425    lastMigration = metaRow;
426    if (metaRow.getVersion() == null) {
427      throw new IllegalStateException("No runVersion in db migration table row? " + metaRow);
428    }
429
430    migrations.put(key, metaRow);
431    if (VERSION_TYPE.equals(metaRow.getType()) || BOOTINIT_TYPE.equals(metaRow.getType())) {
432      MigrationVersion rowVersion = MigrationVersion.parse(metaRow.getVersion());
433      if (currentVersion == null || rowVersion.compareTo(currentVersion) > 0) {
434        currentVersion = rowVersion;
435      }
436      if (BOOTINIT_TYPE.equals(metaRow.getType())) {
437        dbInitVersion = rowVersion;
438      }
439    }
440  }
441
442  /**
443   * Return true if there are no migrations.
444   */
445  public boolean isEmpty() {
446    return migrations.isEmpty();
447  }
448
449  /**
450   * Run all the migrations in order as needed.
451   *
452   * @return the migrations that have been run (collected if checkstate is true).
453   */
454  public List<LocalMigrationResource> runAll(List<LocalMigrationResource> localVersions) throws SQLException {
455
456    checkMinVersion();
457    for (LocalMigrationResource localVersion : localVersions) {
458      if (!localVersion.isRepeatable() && dbInitVersion != null && dbInitVersion.compareTo(localVersion.getVersion()) >= 0) {
459        logger.debug("migration skipped by dbInitVersion {}", dbInitVersion);
460      } else if (!shouldRun(localVersion, priorVersion)) {
461        break;
462      }
463    }
464    return checkMigrations;
465  }
466
467  private void checkMinVersion() {
468    if (minVersion != null && currentVersion != null && currentVersion.compareTo(minVersion) < 0) {
469      StringBuilder sb = new StringBuilder();
470      if (minVersionFailMessage != null && !minVersionFailMessage.isEmpty()) {
471        sb.append(minVersionFailMessage).append(' ');
472      }
473      sb.append("MigrationVersion mismatch: v").append(currentVersion).append(" < v").append(minVersion);
474      throw new MigrationException(sb.toString());
475    }
476  }
477
478  /**
479   * Run using an init migration.
480   *
481   * @return the migrations that have been run (collected if checkstate is true).
482   */
483  public List<LocalMigrationResource> runInit(LocalMigrationResource initVersion, List<LocalMigrationResource> localVersions) throws SQLException {
484
485    runRepeatableInit(localVersions);
486
487    initVersion.setInitType();
488    if (!shouldRun(initVersion, null)) {
489      throw new IllegalStateException("Expected to run init migration but it didn't?");
490    }
491
492    // run any migrations greater that the init migration
493    for (LocalMigrationResource localVersion : localVersions) {
494      if (localVersion.compareTo(initVersion) > 0 && !shouldRun(localVersion, priorVersion)) {
495        break;
496      }
497    }
498    return checkMigrations;
499  }
500
501  private void runRepeatableInit(List<LocalMigrationResource> localVersions) throws SQLException {
502    for (LocalMigrationResource localVersion : localVersions) {
503      if (!localVersion.isRepeatableInit() || !shouldRun(localVersion, priorVersion)) {
504        break;
505      }
506    }
507  }
508
509  /**
510   * Run non transactional statements (if any) after migration commit.
511   * <p>
512   * These run with auto commit true and run AFTER the migration commit and
513   * as such the migration isn't truely atomic - the migration can run and
514   * complete and the non-transactional statements fail.
515   */
516  public void runNonTransactional() {
517    scriptRunner.runNonTransactional();
518  }
519}