001package io.ebean.migration.runner; 002 003import io.ebean.migration.JdbcMigration; 004import io.ebean.migration.MigrationConfig; 005import io.ebean.migration.MigrationException; 006import io.ebean.migration.MigrationVersion; 007import io.ebean.migration.util.IOUtils; 008import org.slf4j.Logger; 009import org.slf4j.LoggerFactory; 010 011import java.io.IOException; 012import java.net.URL; 013import java.sql.Connection; 014import java.sql.DatabaseMetaData; 015import java.sql.ResultSet; 016import java.sql.SQLException; 017import java.sql.Timestamp; 018import java.util.ArrayList; 019import java.util.Enumeration; 020import java.util.LinkedHashMap; 021import java.util.List; 022import java.util.Set; 023 024import static io.ebean.migration.MigrationVersion.BOOTINIT_TYPE; 025import static io.ebean.migration.MigrationVersion.VERSION_TYPE; 026 027/** 028 * Manages the migration table. 029 */ 030public class MigrationTable { 031 032 private static final Logger logger = LoggerFactory.getLogger("io.ebean.DDL"); 033 034 private static final String INIT_VER_0 = "0"; 035 036 private final Connection connection; 037 private final boolean checkState; 038 private final MigrationPlatform platform; 039 private final MigrationScriptRunner scriptRunner; 040 private final String catalog; 041 private final String schema; 042 private final String table; 043 private final String sqlTable; 044 private final String envUserName; 045 private final String platformName; 046 047 private final Timestamp runOn = new Timestamp(System.currentTimeMillis()); 048 049 private final ScriptTransform scriptTransform; 050 051 private final String insertSql; 052 private final String updateSql; 053 private final String updateChecksumSql; 054 055 private final LinkedHashMap<String, MigrationMetaRow> migrations; 056 private final boolean skipChecksum; 057 058 private final Set<String> patchInsertVersions; 059 private final Set<String> patchResetChecksumVersions; 060 private final boolean allowErrorInRepeatable; 061 062 private final MigrationVersion minVersion; 063 private final String minVersionFailMessage; 064 065 private MigrationVersion currentVersion; 066 private MigrationMetaRow lastMigration; 067 private LocalMigrationResource priorVersion; 068 069 private final List<LocalMigrationResource> checkMigrations = new ArrayList<>(); 070 071 /** 072 * Version of a dbinit script. When set this means all migration version less than this are ignored. 073 */ 074 private MigrationVersion dbInitVersion; 075 076 /** 077 * Construct with server, configuration and jdbc connection (DB admin user). 078 */ 079 public MigrationTable(MigrationConfig config, Connection connection, boolean checkState, MigrationPlatform platform) { 080 this.platform = platform; 081 this.connection = connection; 082 this.scriptRunner = new MigrationScriptRunner(connection, platform); 083 this.checkState = checkState; 084 this.migrations = new LinkedHashMap<>(); 085 this.catalog = null; 086 this.allowErrorInRepeatable = config.isAllowErrorInRepeatable(); 087 this.patchResetChecksumVersions = config.getPatchResetChecksumOn(); 088 this.patchInsertVersions = config.getPatchInsertOn(); 089 this.minVersion = initMinVersion(config.getMinVersion()); 090 this.minVersionFailMessage = config.getMinVersionFailMessage(); 091 this.skipChecksum = config.isSkipChecksum(); 092 this.schema = config.getDbSchema(); 093 this.table = config.getMetaTable(); 094 this.platformName = config.getPlatformName(); 095 this.sqlTable = initSqlTable(); 096 this.insertSql = MigrationMetaRow.insertSql(sqlTable); 097 this.updateSql = MigrationMetaRow.updateSql(sqlTable); 098 this.updateChecksumSql = MigrationMetaRow.updateChecksumSql(sqlTable); 099 this.scriptTransform = createScriptTransform(config); 100 this.envUserName = System.getProperty("user.name"); 101 } 102 103 private MigrationVersion initMinVersion(String minVersion) { 104 return (minVersion == null || minVersion.isEmpty()) ? null : MigrationVersion.parse(minVersion); 105 } 106 107 private String initSqlTable() { 108 if (schema != null) { 109 return schema + "." + table; 110 } else { 111 return table; 112 } 113 } 114 115 private String sqlPrimaryKey() { 116 return "pk_" + table; 117 } 118 119 /** 120 * Return the number of migrations in the DB migration table. 121 */ 122 public int size() { 123 return migrations.size(); 124 } 125 126 /** 127 * Returns the versions that are already applied. 128 */ 129 public Set<String> getVersions() { 130 return migrations.keySet(); 131 } 132 133 /** 134 * Create the ScriptTransform for placeholder key/value replacement. 135 */ 136 private ScriptTransform createScriptTransform(MigrationConfig config) { 137 return ScriptTransform.build(config.getRunPlaceholders(), config.getRunPlaceholderMap()); 138 } 139 140 /** 141 * Create the table is it does not exist. 142 * <p> 143 * Also holds DB lock on migration table and loads existing migrations. 144 * </p> 145 */ 146 public void createIfNeededAndLock() throws SQLException, IOException { 147 if (!tableExists()) { 148 createTable(); 149 } 150 obtainLockWithWait(); 151 readExistingMigrations(); 152 } 153 154 /** 155 * Obtain lock with wait, note that other nodes can insert and commit 156 * into the migration table during the wait so this query result won't 157 * contain all the executed migrations in that case. 158 */ 159 private void obtainLockWithWait() throws SQLException { 160 platform.lockMigrationTable(sqlTable, connection); 161 } 162 163 /** 164 * Read the migration table with details on what migrations have run. 165 * This must execute after we have completed the wait for the lock on 166 * the migration table such that it reads any migrations that have 167 * executed during the wait for the lock. 168 */ 169 private void readExistingMigrations() throws SQLException { 170 for (MigrationMetaRow metaRow : platform.readExistingMigrations(sqlTable, connection)) { 171 addMigration(metaRow.getVersion(), metaRow); 172 } 173 } 174 175 private void createTable() throws IOException, SQLException { 176 scriptRunner.runScript(createTableDdl(), "create migration table"); 177 createInitMetaRow().executeInsert(connection, insertSql); 178 } 179 180 /** 181 * Return the create table script. 182 */ 183 String createTableDdl() throws IOException { 184 String script = ScriptTransform.replace("${table}", sqlTable, getCreateTableScript()); 185 return ScriptTransform.replace("${pk_table}", sqlPrimaryKey(), script); 186 } 187 188 /** 189 * Return the create table script. 190 */ 191 private String getCreateTableScript() throws IOException { 192 // supply a script to override the default table create script 193 String script = readResource("migration-support/create-table.sql"); 194 if (script == null && platformName != null && !platformName.isEmpty()) { 195 // look for platform specific create table 196 script = readResource("migration-support/" + platformName + "-create-table.sql"); 197 } 198 if (script == null) { 199 // no, just use the default script 200 script = readResource("migration-support/default-create-table.sql"); 201 } 202 return script; 203 } 204 205 private String readResource(String location) throws IOException { 206 Enumeration<URL> resources = getClassLoader().getResources(location); 207 if (resources.hasMoreElements()) { 208 URL url = resources.nextElement(); 209 return IOUtils.readUtf8(url); 210 } 211 return null; 212 } 213 214 private ClassLoader getClassLoader() { 215 return Thread.currentThread().getContextClassLoader(); 216 } 217 218 /** 219 * Return true if the table exists. 220 */ 221 private boolean tableExists() throws SQLException { 222 String migTable = table; 223 DatabaseMetaData metaData = connection.getMetaData(); 224 if (metaData.storesUpperCaseIdentifiers()) { 225 migTable = migTable.toUpperCase(); 226 } 227 String checkCatalog = (catalog != null) ? catalog : connection.getCatalog(); 228 String checkSchema = (schema != null) ? schema : connection.getSchema(); 229 try (ResultSet tables = metaData.getTables(checkCatalog, checkSchema, migTable, null)) { 230 return tables.next(); 231 } 232 } 233 234 /** 235 * Return true if the migration ran successfully and false if the migration failed. 236 */ 237 private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationResource prior) throws SQLException { 238 if (prior != null && !localVersion.isRepeatable()) { 239 if (!migrationExists(prior)) { 240 logger.error("Migration {} requires prior migration {} which has not been run", localVersion.getVersion(), prior.getVersion()); 241 return false; 242 } 243 } 244 245 MigrationMetaRow existing = migrations.get(localVersion.key()); 246 if (!runMigration(localVersion, existing)) { 247 return false; 248 } 249 250 // migration was run successfully ... 251 priorVersion = localVersion; 252 return true; 253 } 254 255 /** 256 * Run the migration script. 257 * 258 * @param local The local migration resource 259 * @param existing The information for this migration existing in the table 260 * @return True if the migrations should continue 261 */ 262 private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException { 263 String script = null; 264 int checksum; 265 if (local instanceof LocalDdlMigrationResource) { 266 script = convertScript(local.getContent()); 267 checksum = Checksum.calculate(script); 268 } else { 269 checksum = ((LocalJdbcMigrationResource) local).getChecksum(); 270 } 271 272 if (existing == null && patchInsertMigration(local, checksum)) { 273 return true; 274 } 275 if (existing != null && skipMigration(checksum, local, existing)) { 276 return true; 277 } 278 executeMigration(local, script, checksum, existing); 279 return true; 280 } 281 282 /** 283 * Return true if we 'patch history' inserting a DB migration without running it. 284 */ 285 private boolean patchInsertMigration(LocalMigrationResource local, int checksum) throws SQLException { 286 if (patchInsertVersions != null && patchInsertVersions.contains(local.key())) { 287 logger.info("patch migration - insert into history {}", local.getLocation()); 288 if (!checkState) { 289 insertIntoHistory(local, checksum, 0); 290 } 291 return true; 292 } 293 return false; 294 } 295 296 /** 297 * Return true if the migration should be skipped. 298 */ 299 boolean skipMigration(int checksum, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException { 300 boolean matchChecksum = (existing.getChecksum() == checksum); 301 if (matchChecksum) { 302 logger.trace("... skip unchanged migration {}", local.getLocation()); 303 return true; 304 305 } else if (patchResetChecksum(existing, checksum)) { 306 logger.info("patch migration - reset checksum on {}", local.getLocation()); 307 return true; 308 309 } else if (local.isRepeatable() || skipChecksum) { 310 // re-run the migration 311 return false; 312 } else { 313 throw new MigrationException("Checksum mismatch on migration " + local.getLocation()); 314 } 315 } 316 317 /** 318 * Return true if the checksum is reset on the existing migration. 319 */ 320 private boolean patchResetChecksum(MigrationMetaRow existing, int newChecksum) throws SQLException { 321 if (isResetOnVersion(existing.getVersion())) { 322 if (!checkState) { 323 existing.resetChecksum(newChecksum, connection, updateChecksumSql); 324 } 325 return true; 326 } else { 327 return false; 328 } 329 } 330 331 private boolean isResetOnVersion(String version) { 332 return patchResetChecksumVersions != null && patchResetChecksumVersions.contains(version); 333 } 334 335 /** 336 * Run a migration script as new migration or update on existing repeatable migration. 337 */ 338 private void executeMigration(LocalMigrationResource local, String script, int checksum, MigrationMetaRow existing) throws SQLException { 339 if (checkState) { 340 checkMigrations.add(local); 341 // simulate the migration being run such that following migrations also match 342 addMigration(local.key(), createMetaRow(local, checksum, 1)); 343 return; 344 } 345 346 logger.debug("run migration {}", local.getLocation()); 347 348 long start = System.currentTimeMillis(); 349 try { 350 if (local instanceof LocalDdlMigrationResource) { 351 scriptRunner.runScript(script, "run migration version: " + local.getVersion()); 352 } else { 353 JdbcMigration migration = ((LocalJdbcMigrationResource) local).getMigration(); 354 logger.info("Executing jdbc migration version: {} - {}", local.getVersion(), migration); 355 migration.migrate(connection); 356 } 357 long exeMillis = System.currentTimeMillis() - start; 358 359 if (existing != null) { 360 existing.rerun(checksum, exeMillis, envUserName, runOn); 361 existing.executeUpdate(connection, updateSql); 362 363 } else { 364 insertIntoHistory(local, checksum, exeMillis); 365 } 366 } catch (SQLException e) { 367 if (allowErrorInRepeatable && local.isRepeatableLast()) { 368 // log the exception and continue on repeatable migration 369 logger.error("Continue migration with error executing repeatable migration " + local.getVersion(), e); 370 } else { 371 throw e; 372 } 373 } 374 } 375 376 private void insertIntoHistory(LocalMigrationResource local, int checksum, long exeMillis) throws SQLException { 377 MigrationMetaRow metaRow = createMetaRow(local, checksum, exeMillis); 378 metaRow.executeInsert(connection, insertSql); 379 addMigration(local.key(), metaRow); 380 } 381 382 private MigrationMetaRow createInitMetaRow() { 383 return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", 0, envUserName, runOn, 0); 384 } 385 386 /** 387 * Create the MigrationMetaRow for this migration. 388 */ 389 private MigrationMetaRow createMetaRow(LocalMigrationResource migration, int checksum, long exeMillis) { 390 391 int nextId = 1; 392 if (lastMigration != null) { 393 nextId = lastMigration.getId() + 1; 394 } 395 396 String type = migration.getType(); 397 String runVersion = migration.key(); 398 String comment = migration.getComment(); 399 400 return new MigrationMetaRow(nextId, type, runVersion, comment, checksum, envUserName, runOn, exeMillis); 401 } 402 403 /** 404 * Return true if the migration exists. 405 */ 406 private boolean migrationExists(LocalMigrationResource priorVersion) { 407 return migrations.containsKey(priorVersion.key()); 408 } 409 410 /** 411 * Apply the placeholder key/value replacement on the script. 412 */ 413 private String convertScript(String script) { 414 return scriptTransform.transform(script); 415 } 416 417 /** 418 * Register the successfully executed migration (to allow dependant scripts to run). 419 */ 420 private void addMigration(String key, MigrationMetaRow metaRow) { 421 if (INIT_VER_0.equals(key)) { 422 // ignore the version 0 <init> row 423 return; 424 } 425 lastMigration = metaRow; 426 if (metaRow.getVersion() == null) { 427 throw new IllegalStateException("No runVersion in db migration table row? " + metaRow); 428 } 429 430 migrations.put(key, metaRow); 431 if (VERSION_TYPE.equals(metaRow.getType()) || BOOTINIT_TYPE.equals(metaRow.getType())) { 432 MigrationVersion rowVersion = MigrationVersion.parse(metaRow.getVersion()); 433 if (currentVersion == null || rowVersion.compareTo(currentVersion) > 0) { 434 currentVersion = rowVersion; 435 } 436 if (BOOTINIT_TYPE.equals(metaRow.getType())) { 437 dbInitVersion = rowVersion; 438 } 439 } 440 } 441 442 /** 443 * Return true if there are no migrations. 444 */ 445 public boolean isEmpty() { 446 return migrations.isEmpty(); 447 } 448 449 /** 450 * Run all the migrations in order as needed. 451 * 452 * @return the migrations that have been run (collected if checkstate is true). 453 */ 454 public List<LocalMigrationResource> runAll(List<LocalMigrationResource> localVersions) throws SQLException { 455 456 checkMinVersion(); 457 for (LocalMigrationResource localVersion : localVersions) { 458 if (!localVersion.isRepeatable() && dbInitVersion != null && dbInitVersion.compareTo(localVersion.getVersion()) >= 0) { 459 logger.debug("migration skipped by dbInitVersion {}", dbInitVersion); 460 } else if (!shouldRun(localVersion, priorVersion)) { 461 break; 462 } 463 } 464 return checkMigrations; 465 } 466 467 private void checkMinVersion() { 468 if (minVersion != null && currentVersion != null && currentVersion.compareTo(minVersion) < 0) { 469 StringBuilder sb = new StringBuilder(); 470 if (minVersionFailMessage != null && !minVersionFailMessage.isEmpty()) { 471 sb.append(minVersionFailMessage).append(' '); 472 } 473 sb.append("MigrationVersion mismatch: v").append(currentVersion).append(" < v").append(minVersion); 474 throw new MigrationException(sb.toString()); 475 } 476 } 477 478 /** 479 * Run using an init migration. 480 * 481 * @return the migrations that have been run (collected if checkstate is true). 482 */ 483 public List<LocalMigrationResource> runInit(LocalMigrationResource initVersion, List<LocalMigrationResource> localVersions) throws SQLException { 484 485 runRepeatableInit(localVersions); 486 487 initVersion.setInitType(); 488 if (!shouldRun(initVersion, null)) { 489 throw new IllegalStateException("Expected to run init migration but it didn't?"); 490 } 491 492 // run any migrations greater that the init migration 493 for (LocalMigrationResource localVersion : localVersions) { 494 if (localVersion.compareTo(initVersion) > 0 && !shouldRun(localVersion, priorVersion)) { 495 break; 496 } 497 } 498 return checkMigrations; 499 } 500 501 private void runRepeatableInit(List<LocalMigrationResource> localVersions) throws SQLException { 502 for (LocalMigrationResource localVersion : localVersions) { 503 if (!localVersion.isRepeatableInit() || !shouldRun(localVersion, priorVersion)) { 504 break; 505 } 506 } 507 } 508 509 /** 510 * Run non transactional statements (if any) after migration commit. 511 * <p> 512 * These run with auto commit true and run AFTER the migration commit and 513 * as such the migration isn't truely atomic - the migration can run and 514 * complete and the non-transactional statements fail. 515 */ 516 public void runNonTransactional() { 517 scriptRunner.runNonTransactional(); 518 } 519}