001package io.ebean.migration.runner; 002 003import io.ebean.ddlrunner.ScriptTransform; 004import io.ebean.migration.JdbcMigration; 005import io.ebean.migration.MigrationConfig; 006import io.ebean.migration.MigrationException; 007import io.ebean.migration.MigrationVersion; 008import org.slf4j.Logger; 009import org.slf4j.LoggerFactory; 010 011import java.io.IOException; 012import java.net.URL; 013import java.sql.Connection; 014import java.sql.DatabaseMetaData; 015import java.sql.ResultSet; 016import java.sql.SQLException; 017import java.sql.Timestamp; 018import java.util.ArrayList; 019import java.util.Enumeration; 020import java.util.LinkedHashMap; 021import java.util.List; 022import java.util.Set; 023 024import static io.ebean.migration.MigrationVersion.BOOTINIT_TYPE; 025import static io.ebean.migration.MigrationVersion.VERSION_TYPE; 026 027/** 028 * Manages the migration table. 029 */ 030public class MigrationTable { 031 032 private static final Logger logger = LoggerFactory.getLogger("io.ebean.DDL"); 033 034 private static final String INIT_VER_0 = "0"; 035 036 private final Connection connection; 037 private final boolean checkState; 038 private final MigrationPlatform platform; 039 private final MigrationScriptRunner scriptRunner; 040 private final String catalog; 041 private final String schema; 042 private final String table; 043 private final String sqlTable; 044 private final String envUserName; 045 private final String platformName; 046 047 private final Timestamp runOn = new Timestamp(System.currentTimeMillis()); 048 049 private final ScriptTransform scriptTransform; 050 051 private final String insertSql; 052 private final String updateSql; 053 private final String updateChecksumSql; 054 055 private final LinkedHashMap<String, MigrationMetaRow> migrations; 056 private final boolean skipChecksum; 057 private final boolean skipMigrationRun; 058 059 private final Set<String> patchInsertVersions; 060 private final Set<String> patchResetChecksumVersions; 061 private final boolean allowErrorInRepeatable; 062 063 private final MigrationVersion minVersion; 064 private final String minVersionFailMessage; 065 066 private MigrationVersion currentVersion; 067 private MigrationMetaRow lastMigration; 068 private LocalMigrationResource priorVersion; 069 070 private final List<LocalMigrationResource> checkMigrations = new ArrayList<>(); 071 072 /** 073 * Version of a dbinit script. When set this means all migration version less than this are ignored. 074 */ 075 private MigrationVersion dbInitVersion; 076 077 /** 078 * Construct with server, configuration and jdbc connection (DB admin user). 079 */ 080 public MigrationTable(MigrationConfig config, Connection connection, boolean checkState, MigrationPlatform platform) { 081 this.platform = platform; 082 this.connection = connection; 083 this.scriptRunner = new MigrationScriptRunner(connection, platform); 084 this.checkState = checkState; 085 this.migrations = new LinkedHashMap<>(); 086 this.catalog = null; 087 this.allowErrorInRepeatable = config.isAllowErrorInRepeatable(); 088 this.patchResetChecksumVersions = config.getPatchResetChecksumOn(); 089 this.patchInsertVersions = config.getPatchInsertOn(); 090 this.minVersion = initMinVersion(config.getMinVersion()); 091 this.minVersionFailMessage = config.getMinVersionFailMessage(); 092 this.skipMigrationRun = config.isSkipMigrationRun(); 093 this.skipChecksum = config.isSkipChecksum(); 094 this.schema = config.getDbSchema(); 095 this.table = config.getMetaTable(); 096 this.platformName = config.getPlatformName(); 097 this.sqlTable = initSqlTable(); 098 this.insertSql = MigrationMetaRow.insertSql(sqlTable); 099 this.updateSql = MigrationMetaRow.updateSql(sqlTable); 100 this.updateChecksumSql = MigrationMetaRow.updateChecksumSql(sqlTable); 101 this.scriptTransform = createScriptTransform(config); 102 this.envUserName = System.getProperty("user.name"); 103 } 104 105 private MigrationVersion initMinVersion(String minVersion) { 106 return (minVersion == null || minVersion.isEmpty()) ? null : MigrationVersion.parse(minVersion); 107 } 108 109 private String initSqlTable() { 110 if (schema != null) { 111 return schema + "." + table; 112 } else { 113 return table; 114 } 115 } 116 117 private String sqlPrimaryKey() { 118 return "pk_" + table; 119 } 120 121 /** 122 * Return the number of migrations in the DB migration table. 123 */ 124 public int size() { 125 return migrations.size(); 126 } 127 128 /** 129 * Returns the versions that are already applied. 130 */ 131 public Set<String> getVersions() { 132 return migrations.keySet(); 133 } 134 135 /** 136 * Create the ScriptTransform for placeholder key/value replacement. 137 */ 138 private ScriptTransform createScriptTransform(MigrationConfig config) { 139 return ScriptTransform.build(config.getRunPlaceholders(), config.getRunPlaceholderMap()); 140 } 141 142 /** 143 * Create the table is it does not exist. 144 * <p> 145 * Also holds DB lock on migration table and loads existing migrations. 146 * </p> 147 */ 148 public void createIfNeededAndLock() throws SQLException, IOException { 149 if (!tableExists()) { 150 createTable(); 151 } 152 obtainLockWithWait(); 153 readExistingMigrations(); 154 } 155 156 /** 157 * Obtain lock with wait, note that other nodes can insert and commit 158 * into the migration table during the wait so this query result won't 159 * contain all the executed migrations in that case. 160 */ 161 private void obtainLockWithWait() throws SQLException { 162 platform.lockMigrationTable(sqlTable, connection); 163 } 164 165 /** 166 * Read the migration table with details on what migrations have run. 167 * This must execute after we have completed the wait for the lock on 168 * the migration table such that it reads any migrations that have 169 * executed during the wait for the lock. 170 */ 171 private void readExistingMigrations() throws SQLException { 172 for (MigrationMetaRow metaRow : platform.readExistingMigrations(sqlTable, connection)) { 173 addMigration(metaRow.getVersion(), metaRow); 174 } 175 } 176 177 private void createTable() throws IOException, SQLException { 178 scriptRunner.runScript(createTableDdl(), "create migration table"); 179 createInitMetaRow().executeInsert(connection, insertSql); 180 } 181 182 /** 183 * Return the create table script. 184 */ 185 String createTableDdl() throws IOException { 186 String script = ScriptTransform.replace("${table}", sqlTable, getCreateTableScript()); 187 return ScriptTransform.replace("${pk_table}", sqlPrimaryKey(), script); 188 } 189 190 /** 191 * Return the create table script. 192 */ 193 private String getCreateTableScript() throws IOException { 194 // supply a script to override the default table create script 195 String script = readResource("migration-support/create-table.sql"); 196 if (script == null && platformName != null && !platformName.isEmpty()) { 197 // look for platform specific create table 198 script = readResource("migration-support/" + platformName + "-create-table.sql"); 199 } 200 if (script == null) { 201 // no, just use the default script 202 script = readResource("migration-support/default-create-table.sql"); 203 } 204 return script; 205 } 206 207 private String readResource(String location) throws IOException { 208 Enumeration<URL> resources = getClassLoader().getResources(location); 209 if (resources.hasMoreElements()) { 210 URL url = resources.nextElement(); 211 return IOUtils.readUtf8(url); 212 } 213 return null; 214 } 215 216 private ClassLoader getClassLoader() { 217 return Thread.currentThread().getContextClassLoader(); 218 } 219 220 /** 221 * Return true if the table exists. 222 */ 223 private boolean tableExists() throws SQLException { 224 String migTable = table; 225 DatabaseMetaData metaData = connection.getMetaData(); 226 if (metaData.storesUpperCaseIdentifiers()) { 227 migTable = migTable.toUpperCase(); 228 } 229 String checkCatalog = (catalog != null) ? catalog : connection.getCatalog(); 230 String checkSchema = (schema != null) ? schema : connection.getSchema(); 231 try (ResultSet tables = metaData.getTables(checkCatalog, checkSchema, migTable, null)) { 232 return tables.next(); 233 } 234 } 235 236 /** 237 * Return true if the migration ran successfully and false if the migration failed. 238 */ 239 private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationResource prior) throws SQLException { 240 if (prior != null && !localVersion.isRepeatable()) { 241 if (!migrationExists(prior)) { 242 logger.error("Migration {} requires prior migration {} which has not been run", localVersion.getVersion(), prior.getVersion()); 243 return false; 244 } 245 } 246 247 MigrationMetaRow existing = migrations.get(localVersion.key()); 248 if (!runMigration(localVersion, existing)) { 249 return false; 250 } 251 252 // migration was run successfully ... 253 priorVersion = localVersion; 254 return true; 255 } 256 257 /** 258 * Run the migration script. 259 * 260 * @param local The local migration resource 261 * @param existing The information for this migration existing in the table 262 * @return True if the migrations should continue 263 */ 264 private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException { 265 String script = null; 266 int checksum; 267 if (local instanceof LocalDdlMigrationResource) { 268 script = convertScript(local.getContent()); 269 checksum = Checksum.calculate(script); 270 } else { 271 checksum = ((LocalJdbcMigrationResource) local).getChecksum(); 272 } 273 274 if (existing == null && patchInsertMigration(local, checksum)) { 275 return true; 276 } 277 if (existing != null && skipMigration(checksum, local, existing)) { 278 return true; 279 } 280 executeMigration(local, script, checksum, existing); 281 return true; 282 } 283 284 /** 285 * Return true if we 'patch history' inserting a DB migration without running it. 286 */ 287 private boolean patchInsertMigration(LocalMigrationResource local, int checksum) throws SQLException { 288 if (patchInsertVersions != null && patchInsertVersions.contains(local.key())) { 289 logger.info("patch migration - insert into history {}", local.getLocation()); 290 if (!checkState) { 291 insertIntoHistory(local, checksum, 0); 292 } 293 return true; 294 } 295 return false; 296 } 297 298 /** 299 * Return true if the migration should be skipped. 300 */ 301 boolean skipMigration(int checksum, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException { 302 boolean matchChecksum = (existing.getChecksum() == checksum); 303 if (matchChecksum) { 304 logger.trace("... skip unchanged migration {}", local.getLocation()); 305 return true; 306 307 } else if (patchResetChecksum(existing, checksum)) { 308 logger.info("patch migration - reset checksum on {}", local.getLocation()); 309 return true; 310 311 } else if (local.isRepeatable() || skipChecksum) { 312 // re-run the migration 313 return false; 314 } else { 315 throw new MigrationException("Checksum mismatch on migration " + local.getLocation()); 316 } 317 } 318 319 /** 320 * Return true if the checksum is reset on the existing migration. 321 */ 322 private boolean patchResetChecksum(MigrationMetaRow existing, int newChecksum) throws SQLException { 323 if (isResetOnVersion(existing.getVersion())) { 324 if (!checkState) { 325 existing.resetChecksum(newChecksum, connection, updateChecksumSql); 326 } 327 return true; 328 } else { 329 return false; 330 } 331 } 332 333 private boolean isResetOnVersion(String version) { 334 return patchResetChecksumVersions != null && patchResetChecksumVersions.contains(version); 335 } 336 337 /** 338 * Run a migration script as new migration or update on existing repeatable migration. 339 */ 340 private void executeMigration(LocalMigrationResource local, String script, int checksum, MigrationMetaRow existing) throws SQLException { 341 if (checkState) { 342 checkMigrations.add(local); 343 // simulate the migration being run such that following migrations also match 344 addMigration(local.key(), createMetaRow(local, checksum, 1)); 345 return; 346 } 347 348 long exeMillis = 0; 349 try { 350 if (skipMigrationRun) { 351 logger.debug("skip migration {}", local.getLocation()); 352 } else { 353 exeMillis = executeMigration(local, script); 354 } 355 if (existing != null) { 356 existing.rerun(checksum, exeMillis, envUserName, runOn); 357 existing.executeUpdate(connection, updateSql); 358 } else { 359 insertIntoHistory(local, checksum, exeMillis); 360 } 361 } catch (SQLException e) { 362 if (allowErrorInRepeatable && local.isRepeatableLast()) { 363 // log the exception and continue on repeatable migration 364 logger.error("Continue migration with error executing repeatable migration " + local.getVersion(), e); 365 } else { 366 throw e; 367 } 368 } 369 } 370 371 private long executeMigration(LocalMigrationResource local, String script) throws SQLException { 372 long start = System.currentTimeMillis(); 373 if (local instanceof LocalDdlMigrationResource) { 374 logger.debug("run migration {}", local.getLocation()); 375 scriptRunner.runScript(script, "run migration version: " + local.getVersion()); 376 } else { 377 JdbcMigration migration = ((LocalJdbcMigrationResource) local).getMigration(); 378 logger.info("Executing jdbc migration version: {} - {}", local.getVersion(), migration); 379 migration.migrate(connection); 380 } 381 return System.currentTimeMillis() - start; 382 } 383 384 private void insertIntoHistory(LocalMigrationResource local, int checksum, long exeMillis) throws SQLException { 385 MigrationMetaRow metaRow = createMetaRow(local, checksum, exeMillis); 386 metaRow.executeInsert(connection, insertSql); 387 addMigration(local.key(), metaRow); 388 } 389 390 private MigrationMetaRow createInitMetaRow() { 391 return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", 0, envUserName, runOn, 0); 392 } 393 394 /** 395 * Create the MigrationMetaRow for this migration. 396 */ 397 private MigrationMetaRow createMetaRow(LocalMigrationResource migration, int checksum, long exeMillis) { 398 399 int nextId = 1; 400 if (lastMigration != null) { 401 nextId = lastMigration.getId() + 1; 402 } 403 404 String type = migration.getType(); 405 String runVersion = migration.key(); 406 String comment = migration.getComment(); 407 408 return new MigrationMetaRow(nextId, type, runVersion, comment, checksum, envUserName, runOn, exeMillis); 409 } 410 411 /** 412 * Return true if the migration exists. 413 */ 414 private boolean migrationExists(LocalMigrationResource priorVersion) { 415 return migrations.containsKey(priorVersion.key()); 416 } 417 418 /** 419 * Apply the placeholder key/value replacement on the script. 420 */ 421 private String convertScript(String script) { 422 return scriptTransform.transform(script); 423 } 424 425 /** 426 * Register the successfully executed migration (to allow dependant scripts to run). 427 */ 428 private void addMigration(String key, MigrationMetaRow metaRow) { 429 if (INIT_VER_0.equals(key)) { 430 // ignore the version 0 <init> row 431 return; 432 } 433 lastMigration = metaRow; 434 if (metaRow.getVersion() == null) { 435 throw new IllegalStateException("No runVersion in db migration table row? " + metaRow); 436 } 437 438 migrations.put(key, metaRow); 439 if (VERSION_TYPE.equals(metaRow.getType()) || BOOTINIT_TYPE.equals(metaRow.getType())) { 440 MigrationVersion rowVersion = MigrationVersion.parse(metaRow.getVersion()); 441 if (currentVersion == null || rowVersion.compareTo(currentVersion) > 0) { 442 currentVersion = rowVersion; 443 } 444 if (BOOTINIT_TYPE.equals(metaRow.getType())) { 445 dbInitVersion = rowVersion; 446 } 447 } 448 } 449 450 /** 451 * Return true if there are no migrations. 452 */ 453 public boolean isEmpty() { 454 return migrations.isEmpty(); 455 } 456 457 /** 458 * Run all the migrations in order as needed. 459 * 460 * @return the migrations that have been run (collected if checkstate is true). 461 */ 462 public List<LocalMigrationResource> runAll(List<LocalMigrationResource> localVersions) throws SQLException { 463 464 checkMinVersion(); 465 for (LocalMigrationResource localVersion : localVersions) { 466 if (!localVersion.isRepeatable() && dbInitVersion != null && dbInitVersion.compareTo(localVersion.getVersion()) >= 0) { 467 logger.debug("migration skipped by dbInitVersion {}", dbInitVersion); 468 } else if (!shouldRun(localVersion, priorVersion)) { 469 break; 470 } 471 } 472 return checkMigrations; 473 } 474 475 private void checkMinVersion() { 476 if (minVersion != null && currentVersion != null && currentVersion.compareTo(minVersion) < 0) { 477 StringBuilder sb = new StringBuilder(); 478 if (minVersionFailMessage != null && !minVersionFailMessage.isEmpty()) { 479 sb.append(minVersionFailMessage).append(' '); 480 } 481 sb.append("MigrationVersion mismatch: v").append(currentVersion).append(" < v").append(minVersion); 482 throw new MigrationException(sb.toString()); 483 } 484 } 485 486 /** 487 * Run using an init migration. 488 * 489 * @return the migrations that have been run (collected if checkstate is true). 490 */ 491 public List<LocalMigrationResource> runInit(LocalMigrationResource initVersion, List<LocalMigrationResource> localVersions) throws SQLException { 492 493 runRepeatableInit(localVersions); 494 495 initVersion.setInitType(); 496 if (!shouldRun(initVersion, null)) { 497 throw new IllegalStateException("Expected to run init migration but it didn't?"); 498 } 499 500 // run any migrations greater that the init migration 501 for (LocalMigrationResource localVersion : localVersions) { 502 if (localVersion.compareTo(initVersion) > 0 && !shouldRun(localVersion, priorVersion)) { 503 break; 504 } 505 } 506 return checkMigrations; 507 } 508 509 private void runRepeatableInit(List<LocalMigrationResource> localVersions) throws SQLException { 510 for (LocalMigrationResource localVersion : localVersions) { 511 if (!localVersion.isRepeatableInit() || !shouldRun(localVersion, priorVersion)) { 512 break; 513 } 514 } 515 } 516 517 /** 518 * Run non transactional statements (if any) after migration commit. 519 * <p> 520 * These run with auto commit true and run AFTER the migration commit and 521 * as such the migration isn't truely atomic - the migration can run and 522 * complete and the non-transactional statements fail. 523 */ 524 public void runNonTransactional() { 525 scriptRunner.runNonTransactional(); 526 } 527}