001package io.ebeaninternal.server.autotune.service;
002
003import io.ebean.config.AutoTuneConfig;
004import io.ebean.config.DatabaseConfig;
005import io.ebeaninternal.api.SpiEbeanServer;
006import io.ebeaninternal.api.SpiQuery;
007import io.ebeaninternal.server.autotune.AutoTuneService;
008import io.ebeaninternal.server.autotune.model.Autotune;
009import io.ebeaninternal.server.autotune.model.Origin;
010import org.slf4j.Logger;
011import org.slf4j.LoggerFactory;
012
013import java.io.File;
014import java.io.IOException;
015import java.io.InputStream;
016import java.util.concurrent.TimeUnit;
017import java.util.concurrent.locks.ReentrantLock;
018
019/**
020 * Implementation of the AutoTuneService which is comprised of profiling and query tuning.
021 */
022public class DefaultAutoTuneService implements AutoTuneService {
023
024  private static final Logger logger = LoggerFactory.getLogger(DefaultAutoTuneService.class);
025
026  private final ReentrantLock lock = new ReentrantLock();
027
028  private final SpiEbeanServer server;
029
030  private final long defaultGarbageCollectionWait;
031
032  private final boolean skipGarbageCollectionOnShutdown;
033
034  private final boolean skipProfileReportingOnShutdown;
035
036  private final BaseQueryTuner queryTuner;
037
038  private final ProfileManager profileManager;
039
040  private final boolean profiling;
041
042  private final boolean queryTuning;
043
044  private final String tuningFile;
045
046  private final String profilingFile;
047
048  private final String serverName;
049
050  private final int profilingUpdateFrequency;
051
052  private long runtimeChangeCount;
053
054  public DefaultAutoTuneService(SpiEbeanServer server, DatabaseConfig databaseConfig) {
055    AutoTuneConfig config = databaseConfig.getAutoTuneConfig();
056    this.server = server;
057    this.queryTuning = config.isQueryTuning();
058    this.profiling = config.isProfiling();
059    this.tuningFile = config.getQueryTuningFile();
060    this.profilingFile = config.getProfilingFile();
061    this.profilingUpdateFrequency = config.getProfilingUpdateFrequency();
062    this.serverName = server.getName();
063    this.profileManager = new ProfileManager(config, server);
064    this.queryTuner = new BaseQueryTuner(config, server, profileManager);
065    this.skipGarbageCollectionOnShutdown = config.isSkipGarbageCollectionOnShutdown();
066    this.skipProfileReportingOnShutdown = config.isSkipProfileReportingOnShutdown();
067    this.defaultGarbageCollectionWait = config.getGarbageCollectionWait();
068  }
069
070  /**
071   * Load the query tuning information from it's data store.
072   */
073  @Override
074  public void startup() {
075
076    if (queryTuning) {
077      loadTuningFile();
078      if (isRuntimeTuningUpdates()) {
079        // periodically gather and update query tuning
080        server.getBackgroundExecutor().scheduleWithFixedDelay(new ProfilingUpdate(), profilingUpdateFrequency, profilingUpdateFrequency, TimeUnit.SECONDS);
081      }
082    }
083  }
084
085
086  /**
087   * Return true if the tuning should update periodically at runtime.
088   */
089  private boolean isRuntimeTuningUpdates() {
090    return profilingUpdateFrequency > 0;
091  }
092
093  private class ProfilingUpdate implements Runnable {
094
095    @Override
096    public void run() {
097      runtimeTuningUpdate();
098    }
099  }
100
101  /**
102   * Load tuning information from an existing tuning file.
103   */
104  private void loadTuningFile() {
105    File file = new File(tuningFile);
106    if (file.exists()) {
107      loadAutoTuneProfiling(AutoTuneXmlReader.read(file));
108    } else {
109      // look for autotune as a resource
110      try (InputStream stream = getClass().getResourceAsStream("/" + tuningFile)) {
111        if (stream != null) {
112          loadAutoTuneProfiling(AutoTuneXmlReader.read(stream));
113        } else {
114          logger.warn("AutoTune file {} not found - no initial automatic query tuning", tuningFile);
115        }
116      } catch (IOException e) {
117        throw new IllegalStateException("Error on auto close of " + tuningFile, e);
118      }
119    }
120  }
121
122  private void loadAutoTuneProfiling(Autotune profiling) {
123    logger.info("AutoTune loading {} tuning entries", profiling.getOrigin().size());
124    for (Origin origin : profiling.getOrigin()) {
125      queryTuner.put(origin);
126    }
127  }
128
129  /**
130   * Collect profiling, check for new/diff to existing tuning and apply changes.
131   */
132  private void runtimeTuningUpdate() {
133    lock.lock();
134    try {
135      try {
136        long start = System.currentTimeMillis();
137
138        AutoTuneCollection profiling = profileManager.profilingCollection(false);
139
140        AutoTuneDiffCollection event = new AutoTuneDiffCollection(profiling, queryTuner, true);
141        event.process();
142        if (event.isEmpty()) {
143          long exeMillis = System.currentTimeMillis() - start;
144          logger.debug("No query tuning updates for server:{} executionMillis:{}", serverName, exeMillis);
145
146        } else {
147          // report the query tuning changes that have been made
148          runtimeChangeCount += event.getChangeCount();
149          event.writeFile(profilingFile + "-" + serverName + "-update");
150          long exeMillis = System.currentTimeMillis() - start;
151          logger.info("query tuning updates - new:{} diff:{} for server:{} executionMillis:{}", event.getNewCount(), event.getDiffCount(), serverName, exeMillis);
152        }
153      } catch (Throwable e) {
154        logger.error("Error collecting or applying automatic query tuning", e);
155      }
156    } finally {
157      lock.unlock();
158    }
159  }
160
161  private void saveProfilingOnShutdown(boolean reset) {
162    lock.lock();
163    try {
164      if (isRuntimeTuningUpdates()) {
165        runtimeTuningUpdate();
166        outputAllTuning();
167
168      } else {
169
170        AutoTuneCollection profiling = profileManager.profilingCollection(reset);
171
172        AutoTuneDiffCollection event = new AutoTuneDiffCollection(profiling, queryTuner, false);
173        event.process();
174        if (event.isEmpty()) {
175          logger.info("No new or diff entries for profiling server:{}", serverName);
176
177        } else {
178          event.writeFile(profilingFile + "-" + serverName);
179          logger.info("writing new:{} diff:{} profiling entries for server:{}", event.getNewCount(), event.getDiffCount(), serverName);
180        }
181      }
182    } finally {
183      lock.unlock();
184    }
185  }
186
187  /**
188   * Output all the query tuning (the "all" file).
189   * <p>
190   * This is the originally loaded tuning plus any tuning changes picked up and applied at runtime.
191   * </p>
192   * <p>
193   * This "all" file can be used as the next "ebean-autotune.xml" file.
194   * </p>
195   */
196  private void outputAllTuning() {
197
198    if (runtimeChangeCount == 0) {
199      logger.info("no runtime query tuning changes for server:{}", serverName);
200
201    } else {
202      AutoTuneAllCollection event = new AutoTuneAllCollection(queryTuner);
203      int size = event.size();
204      File existingTuning = new File(tuningFile);
205      if (existingTuning.exists()) {
206        // rename the existing autotune.xml file (appending 'now')
207        if (!existingTuning.renameTo(new File(tuningFile + "." + AutoTuneXmlWriter.now()))) {
208          logger.warn("Failed to rename autotune file [{}]", tuningFile);
209        }
210      }
211
212      event.writeFile(tuningFile, false);
213      logger.info("query tuning detected [{}] changes, writing all [{}] tuning entries for server:{}", runtimeChangeCount, size, serverName);
214    }
215  }
216
217  /**
218   * Shutdown the listener.
219   * <p>
220   * We should try to collect the usage statistics by calling a System.gc().
221   * This is necessary for use with short lived applications where garbage
222   * collection may not otherwise occur at all.
223   * </p>
224   */
225  @Override
226  public void shutdown() {
227    if (profiling) {
228      if (!skipGarbageCollectionOnShutdown && !skipProfileReportingOnShutdown) {
229        // trigger GC to update profiling information on recently executed queries
230        collectProfiling(-1);
231      }
232      if (!skipProfileReportingOnShutdown) {
233        saveProfilingOnShutdown(false);
234      }
235    }
236  }
237
238  /**
239   * Output the profiling.
240   * <p>
241   * When profiling updates are applied to tuning at runtime this reports all tuning and profiling combined.
242   * When profiling is not applied at runtime then this reports the diff report with new and diff entries relative
243   * to the existing tuning.
244   * </p>
245   */
246  @Override
247  public void reportProfiling() {
248    saveProfilingOnShutdown(false);
249  }
250
251  /**
252   * Ask for a System.gc() so that we gather node usage information.
253   * <p>
254   * Really only want to do this sparingly but useful just prior to shutdown
255   * for short run application where garbage collection may otherwise not
256   * occur at all.
257   * </p>
258   * <p>
259   * waitMillis will do a thread sleep to give the garbage collection a little
260   * time to do its thing assuming we are shutting down the VM.
261   * </p>
262   * <p>
263   * If waitMillis is -1 then the defaultGarbageCollectionWait is used which
264   * defaults to 100 milliseconds.
265   * </p>
266   */
267  @Override
268  public void collectProfiling() {
269    collectProfiling(-1);
270  }
271
272  public void collectProfiling(long waitMillis) {
273    System.gc();
274    try {
275      if (waitMillis < 0) {
276        waitMillis = defaultGarbageCollectionWait;
277      }
278      Thread.sleep(waitMillis);
279    } catch (InterruptedException e) {
280      // restore the interrupted status
281      Thread.currentThread().interrupt();
282      logger.warn("Error while sleeping after System.gc() request.", e);
283    }
284  }
285
286  /**
287   * Auto tune the query and enable profiling.
288   */
289  @Override
290  public boolean tuneQuery(SpiQuery<?> query) {
291    return queryTuner.tuneQuery(query);
292  }
293
294}