001/* 002 * The contents of this file are subject to the terms of the Common Development and 003 * Distribution License (the License). You may not use this file except in compliance with the 004 * License. 005 * 006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the 007 * specific language governing permission and limitations under the License. 008 * 009 * When distributing Covered Software, include this CDDL Header Notice in each file and include 010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL 011 * Header, with the fields enclosed by brackets [] replaced by your own identifying 012 * information: "Portions copyright [year] [name of copyright owner]". 013 * 014 * Copyright 2015 ForgeRock AS. 015 */ 016 017package org.forgerock.audit.events.handlers.writers; 018 019import static java.util.concurrent.TimeUnit.MILLISECONDS; 020 021import java.io.BufferedWriter; 022import java.io.File; 023import java.io.FileOutputStream; 024import java.io.IOException; 025import java.io.OutputStreamWriter; 026import java.nio.charset.StandardCharsets; 027import java.util.HashSet; 028import java.util.LinkedList; 029import java.util.List; 030import java.util.Set; 031import java.util.concurrent.Executors; 032import java.util.concurrent.ScheduledExecutorService; 033import java.util.concurrent.TimeUnit; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.locks.ReentrantReadWriteLock; 036import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; 037 038import org.forgerock.audit.events.handlers.FileBasedEventHandlerConfiguration; 039import org.forgerock.audit.events.handlers.FileBasedEventHandlerConfiguration.FileRetention; 040import org.forgerock.audit.events.handlers.FileBasedEventHandlerConfiguration.FileRotation; 041import org.forgerock.audit.retention.DiskSpaceUsedRetentionPolicy; 042import org.forgerock.audit.retention.FileNamingPolicy; 043import org.forgerock.audit.retention.FreeDiskSpaceRetentionPolicy; 044import org.forgerock.audit.retention.RetentionPolicy; 045import org.forgerock.audit.retention.SizeBasedRetentionPolicy; 046import org.forgerock.audit.retention.TimeStampFileNamingPolicy; 047import org.forgerock.audit.rotation.FixedTimeRotationPolicy; 048import org.forgerock.audit.rotation.RotatableObject; 049import org.forgerock.audit.rotation.RotationContext; 050import org.forgerock.audit.rotation.RotationHooks; 051import org.forgerock.audit.rotation.RotationPolicy; 052import org.forgerock.audit.rotation.SizeBasedRotationPolicy; 053import org.forgerock.audit.rotation.TimeLimitRotationPolicy; 054import org.forgerock.util.annotations.VisibleForTesting; 055import org.forgerock.util.time.Duration; 056import org.joda.time.DateTime; 057import org.joda.time.DateTimeZone; 058import org.slf4j.Logger; 059import org.slf4j.LoggerFactory; 060 061/** 062 * Creates an {@link RotatableWriter} that supports file rotation and retention. 063 */ 064public class RotatableWriter implements TextWriter, RotatableObject { 065 066 private static final Logger logger = LoggerFactory.getLogger(RotatableWriter.class); 067 private static final Duration ZERO = Duration.duration("zero"); 068 private static final Duration FIVE_SECONDS = Duration.duration("5s"); 069 070 private final List<RotationPolicy> rotationPolicies = new LinkedList<>(); 071 private final List<RetentionPolicy> retentionPolicies = new LinkedList<>(); 072 private final FileNamingPolicy fileNamingPolicy; 073 private ScheduledExecutorService rotator; 074 private DateTime lastRotationTime; 075 private final boolean rotationEnabled; 076 private final File file; 077 private RotationHooks rotationHooks = new RotationHooks.NoOpRotatationHooks(); 078 private final AtomicBoolean isRotating = new AtomicBoolean(false); 079 /** The underlying output stream. */ 080 private MeteredStream meteredStream; 081 /** The underlying buffered writer using the output stream. */ 082 private BufferedWriter writer; 083 private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); 084 private final RolloverLifecycleHook rolloverLifecycleHook; 085 086 /** 087 * Constructs a {@link RotatableWriter} given an initial file to manage rotation/retention, and 088 * a {@link FileBasedEventHandlerConfiguration} 089 * @param file The initial file to manage rotation/retention. 090 * @param configuration The configuration of the rotation and retention policies. 091 * @param append Whether to append to the rotatable file or not. 092 * @throws IOException If a problem occurs. 093 */ 094 public RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration, 095 final boolean append) throws IOException { 096 this(file, configuration, append, getTimeStampFileNamingPolicy(file, configuration)); 097 } 098 099 /** 100 * Constructs a {@link RotatableWriter} given an initial file to manage rotation/retention, a 101 * a {@link FileBasedEventHandlerConfiguration} and a {@link RolloverLifecycleHook} 102 * 103 * @param file The initial file to manage rotation/retention. 104 * @param configuration The configuration of the rotation and retention policies. 105 * @param append Whether to append to the rotatable file or not. 106 * @param rolloverLifecycleHook Hook to use before and after rotation/retention checks. 107 * @throws IOException If a problem occurs. 108 */ 109 public RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration, 110 final boolean append, final RolloverLifecycleHook rolloverLifecycleHook) throws IOException { 111 this(file, configuration, append, getTimeStampFileNamingPolicy(file, configuration), rolloverLifecycleHook); 112 } 113 114 /** 115 * This constructor allows tests to set an alternative FileNamingPolicy as TimeStampFileNamingPolicy lists files 116 * for deletion by their last modified timestamps but these timestamps are only accurate to the nearest second. 117 */ 118 @VisibleForTesting 119 RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration, 120 final boolean append, final FileNamingPolicy fileNamingPolicy) throws IOException { 121 this(file, configuration, append, fileNamingPolicy, NOOP_ROLLOVER_LIFECYCLE_HOOK); 122 } 123 124 /** Constructor with all possible parameters. */ 125 private RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration, 126 final boolean append, final FileNamingPolicy fileNamingPolicy, 127 final RolloverLifecycleHook rolloverLifecycleHook) throws IOException { 128 this.file = file; 129 this.fileNamingPolicy = fileNamingPolicy; 130 this.rotationEnabled = configuration.getFileRotation().isRotationEnabled(); 131 final long lastModified = file.lastModified(); 132 this.lastRotationTime = lastModified > 0 133 ? new DateTime(file.lastModified(), DateTimeZone.UTC) 134 : DateTime.now(DateTimeZone.UTC); 135 this.rolloverLifecycleHook = rolloverLifecycleHook; 136 this.writer = constructWriter(file, append); 137 addRetentionPolicies(configuration.getFileRetention()); 138 addRotationPolicies(configuration.getFileRotation()); 139 scheduleRotationAndRetentionChecks(configuration); 140 } 141 142 private static TimeStampFileNamingPolicy getTimeStampFileNamingPolicy(final File file, 143 final FileBasedEventHandlerConfiguration configuration) { 144 return new TimeStampFileNamingPolicy( 145 file, 146 configuration.getFileRotation().getRotationFileSuffix(), 147 configuration.getFileRotation().getRotationFilePrefix()); 148 } 149 150 /** 151 * Rotate the log file if any of the configured rotation policies determine that rotation is required. 152 * 153 * @throws IOException If unable to rotate the log file. 154 */ 155 @Override 156 public void rotateIfNeeded() throws IOException { 157 if (!rotationEnabled || isRotating.get()) { 158 return; 159 } 160 readWriteLock.writeLock().lock(); 161 try { 162 for (RotationPolicy rotationPolicy : rotationPolicies) { 163 if (rotationPolicy.shouldRotateFile(this)) { 164 if (logger.isTraceEnabled()) { 165 logger.trace("Must rotate: {}", file.getAbsolutePath()); 166 } 167 isRotating.set(true); 168 if (rotate()) { 169 if (logger.isTraceEnabled()) { 170 logger.trace("Finished rotation for: {}", file.getAbsolutePath()); 171 } 172 } 173 break; 174 } 175 } 176 } finally { 177 readWriteLock.writeLock().unlock(); 178 isRotating.set(false); 179 } 180 } 181 182 /** Delete files if they need to be deleted as per enabled retention policies. */ 183 private void deleteFilesIfNeeded() throws IOException { 184 readWriteLock.writeLock().lock(); 185 try { 186 Set<File> filesToDelete = checkRetention(); // return the files to delete, but do not delete them 187 if (!filesToDelete.isEmpty()) { 188 deleteFiles(filesToDelete); 189 } 190 } finally { 191 readWriteLock.writeLock().unlock(); 192 } 193 } 194 195 private boolean rotate() throws IOException { 196 boolean rotationHappened = false; 197 RotationContext context = new RotationContext(); 198 context.setWriter(writer); 199 File currentFile = fileNamingPolicy.getInitialName(); 200 context.setInitialFile(currentFile); 201 if (currentFile.exists()) { 202 File newFile = fileNamingPolicy.getNextName(); 203 context.setNextFile(newFile); 204 rotationHooks.preRotationAction(context); 205 writer.close(); 206 if (logger.isTraceEnabled()) { 207 logger.trace("Renaming {} to {}", currentFile.getAbsolutePath(), newFile.getAbsolutePath()); 208 } 209 if (currentFile.renameTo(newFile)) { 210 rotationHappened = true; 211 if (currentFile.createNewFile()) { 212 writer = constructWriter(currentFile, true); 213 context.setWriter(writer); 214 rotationHooks.postRotationAction(context); 215 } else { 216 logger.error("Unable to resume writing to audit file {}; further events will not be logged", 217 currentFile.toString()); 218 } 219 } else { 220 logger.error("Unable to rename the audit file {}; further events will continue to be logged to " + 221 "the current file", currentFile.toString()); 222 writer = constructWriter(currentFile, true); 223 } 224 lastRotationTime = DateTime.now(DateTimeZone.UTC); 225 } 226 return rotationHappened; 227 } 228 229 private Set<File> checkRetention() throws IOException { 230 Set<File> filesToDelete = new HashSet<>(); 231 for (RetentionPolicy retentionPolicy : retentionPolicies) { 232 filesToDelete.addAll(retentionPolicy.deleteFiles(fileNamingPolicy)); 233 } 234 return filesToDelete; 235 } 236 237 private void deleteFiles(final Set<File> files) { 238 for (final File file : files) { 239 if (logger.isInfoEnabled()) { 240 logger.info("Deleting file {}", file.getAbsolutePath()); 241 } 242 if (!file.delete()) { 243 if (logger.isWarnEnabled()) { 244 logger.warn("Could not delete file {}", file.getAbsolutePath()); 245 } 246 } 247 } 248 } 249 250 /** 251 * {@inheritDoc} 252 */ 253 @Override 254 public long getBytesWritten() { 255 logger.trace("bytes written={}", meteredStream.getBytesWritten()); 256 return meteredStream.getBytesWritten(); 257 } 258 259 /** 260 * {@inheritDoc} 261 */ 262 @Override 263 public DateTime getLastRotationTime() { 264 return lastRotationTime; 265 } 266 267 /** 268 * {@inheritDoc} 269 */ 270 @Override 271 public void close() throws IOException { 272 if (rotator != null) { 273 boolean interrupted = false; 274 rotator.shutdown(); 275 try { 276 while (!rotator.awaitTermination(500, MILLISECONDS)) { 277 logger.debug("Waiting to terminate the rotator thread."); 278 } 279 } catch (InterruptedException ex) { 280 logger.error("Unable to terminate the rotator thread", ex); 281 interrupted = true; 282 } finally { 283 if (interrupted) { 284 Thread.currentThread().interrupt(); 285 } 286 } 287 } 288 writer.close(); 289 } 290 291 @Override 292 public void shutdown() { 293 try { 294 close(); 295 } catch (IOException e) { 296 logger.error("Error when performing shutdown", e); 297 } 298 } 299 300 /** 301 * {@inheritDoc} 302 */ 303 @Override 304 public void registerRotationHooks(final RotationHooks rotationHooks) { 305 this.rotationHooks = rotationHooks; 306 } 307 308 @Override 309 public void write(String str) throws IOException { 310 ReadLock lock = readWriteLock.readLock(); 311 try { 312 lock.lock(); 313 logger.trace("Actually writing to file: {}", str); 314 writer.write(str); 315 } finally { 316 lock.unlock(); 317 } 318 rotateIfNeeded(); 319 } 320 321 /** 322 * Forces a rotation of the writer. 323 * 324 * @return {@code true} if rotation was done, {@code false} otherwise. 325 * @throws IOException 326 * If an error occurs 327 */ 328 public boolean forceRotation() throws IOException { 329 readWriteLock.writeLock().lock(); 330 try { 331 isRotating.set(true); 332 return rotate(); 333 } 334 finally { 335 isRotating.set(false); 336 readWriteLock.writeLock().unlock(); 337 } 338 } 339 340 @Override 341 public void flush() throws IOException { 342 writer.flush(); 343 } 344 345 private BufferedWriter constructWriter(File csvFile, boolean append) 346 throws IOException { 347 FileOutputStream stream = new FileOutputStream(csvFile, append); 348 meteredStream = new MeteredStream(stream, file.length()); 349 OutputStreamWriter osw = new OutputStreamWriter(meteredStream, StandardCharsets.UTF_8); 350 return new BufferedWriter(osw); 351 } 352 353 private void addRotationPolicies(final FileRotation fileRotation) { 354 // add SizeBasedRotationPolicy if a non zero size is supplied 355 final long maxFileSize = fileRotation.getMaxFileSize(); 356 if (maxFileSize > 0) { 357 rotationPolicies.add(new SizeBasedRotationPolicy(maxFileSize)); 358 } 359 360 // add FixedTimeRotationPolicy 361 final List<Duration> dailyRotationTimes = new LinkedList<>(); 362 for (final String rotationTime : fileRotation.getRotationTimes()) { 363 Duration duration = parseDuration("rotation time", rotationTime, null); 364 if (duration != null && !duration.isUnlimited()) { 365 dailyRotationTimes.add(duration); 366 } 367 } 368 if (!dailyRotationTimes.isEmpty()) { 369 rotationPolicies.add(new FixedTimeRotationPolicy(dailyRotationTimes)); 370 } 371 372 // add TimeLimitRotationPolicy if enabled 373 final Duration rotationInterval = parseDuration("rotation interval", fileRotation.getRotationInterval(), ZERO); 374 if (!(rotationInterval.isZero() || rotationInterval.isUnlimited())) { 375 rotationPolicies.add(new TimeLimitRotationPolicy(rotationInterval)); 376 } 377 } 378 379 /** 380 * Schedule checks for rotations and retention policies. 381 * <p> 382 * The check interval is provided by the RotationRetentionCheckInterval property, which must have 383 * a non-zero value if at least one policy is enabled. 384 */ 385 private void scheduleRotationAndRetentionChecks(FileBasedEventHandlerConfiguration configuration) 386 throws IOException { 387 final Duration rotationCheckInterval = parseDuration("rotation and retention check interval", 388 configuration.getRotationRetentionCheckInterval(), FIVE_SECONDS); 389 390 if (!rotationPolicies.isEmpty() || !retentionPolicies.isEmpty()) { 391 if (rotationCheckInterval.isUnlimited() || rotationCheckInterval.isZero()) { 392 throw new IOException("Rotation and retention check interval set to an invalid value: " 393 + rotationCheckInterval); 394 } 395 rotator = Executors.newScheduledThreadPool(1); 396 rotator.scheduleAtFixedRate( 397 new Runnable() { 398 @Override 399 public void run() { 400 rolloverLifecycleHook.beforeRollingOver(); 401 try { 402 try { 403 rotateIfNeeded(); 404 } catch (Exception e) { 405 logger.error("Failure when applying a rotation policy to file {}", 406 fileNamingPolicy.getInitialName(), e); 407 } 408 try { 409 deleteFilesIfNeeded(); 410 } catch (Exception e) { 411 logger.error("Failure when applying a retention policy to file {}", 412 fileNamingPolicy.getInitialName(), e); 413 } 414 } finally { 415 rolloverLifecycleHook.afterRollingOver(); 416 } 417 } 418 }, 419 rotationCheckInterval.to(TimeUnit.MILLISECONDS), 420 rotationCheckInterval.to(TimeUnit.MILLISECONDS), 421 TimeUnit.MILLISECONDS); 422 } 423 } 424 425 private Duration parseDuration(String description, String duration, Duration defaultValue) { 426 try { 427 return Duration.duration(duration); 428 } catch (IllegalArgumentException e) { 429 logger.warn("Invalid {} value: '{}'", description, duration); 430 return defaultValue; 431 } 432 } 433 434 @VisibleForTesting 435 List<RotationPolicy> getRotationPolicies() { 436 return rotationPolicies; 437 } 438 439 private void addRetentionPolicies(final FileRetention fileRetention) { 440 // Add SizeBasedRetentionPolicy if the max number of files config value is more than 0 441 final int maxNumberOfHistoryFiles = fileRetention.getMaxNumberOfHistoryFiles(); 442 if (maxNumberOfHistoryFiles > 0) { 443 retentionPolicies.add(new SizeBasedRetentionPolicy(maxNumberOfHistoryFiles)); 444 } 445 446 // Add DiskSpaceUsedRetentionPolicy if config value > 0 447 final long maxDiskSpaceToUse = fileRetention.getMaxDiskSpaceToUse(); 448 if (maxDiskSpaceToUse > 0) { 449 retentionPolicies.add(new DiskSpaceUsedRetentionPolicy(maxDiskSpaceToUse)); 450 } 451 452 // Add FreeDiskSpaceRetentionPolicy if config value > 0 453 final long minimumFreeDiskSpace = fileRetention.getMinFreeSpaceRequired(); 454 if (minimumFreeDiskSpace > 0) { 455 retentionPolicies.add(new FreeDiskSpaceRetentionPolicy(minimumFreeDiskSpace)); 456 } 457 } 458 459 /** 460 * A RotationRetentionCheckHook that does nothing. 461 */ 462 public static final RolloverLifecycleHook NOOP_ROLLOVER_LIFECYCLE_HOOK = 463 new RolloverLifecycleHook() { 464 @Override 465 public void beforeRollingOver() { 466 // nothing to do 467 } 468 469 @Override 470 public void afterRollingOver() { 471 // nothing to do 472 } 473 }; 474 475 /** 476 * Callback hooks to allow custom action to be taken before and after the checks for rotation and 477 * retention is performed. 478 */ 479 public interface RolloverLifecycleHook { 480 481 /** 482 * This method is called before the rotation and retention checks are done. 483 */ 484 void beforeRollingOver(); 485 486 /** 487 * This method is called after the rotation and retention checks are done. 488 */ 489 void afterRollingOver(); 490 } 491 492}