001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2015-2016 ForgeRock AS.
015 */
016
017package org.forgerock.audit.events.handlers.writers;
018
019import static java.util.concurrent.TimeUnit.MILLISECONDS;
020
021import java.io.BufferedWriter;
022import java.io.File;
023import java.io.FileOutputStream;
024import java.io.IOException;
025import java.io.OutputStreamWriter;
026import java.nio.charset.StandardCharsets;
027import java.util.HashSet;
028import java.util.LinkedList;
029import java.util.List;
030import java.util.Set;
031import java.util.concurrent.Executors;
032import java.util.concurrent.ScheduledExecutorService;
033import java.util.concurrent.TimeUnit;
034import java.util.concurrent.atomic.AtomicBoolean;
035import java.util.concurrent.locks.ReentrantReadWriteLock;
036import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
037
038import org.forgerock.audit.events.handlers.FileBasedEventHandlerConfiguration;
039import org.forgerock.audit.events.handlers.FileBasedEventHandlerConfiguration.FileRetention;
040import org.forgerock.audit.events.handlers.FileBasedEventHandlerConfiguration.FileRotation;
041import org.forgerock.audit.retention.DiskSpaceUsedRetentionPolicy;
042import org.forgerock.audit.retention.FileNamingPolicy;
043import org.forgerock.audit.retention.FreeDiskSpaceRetentionPolicy;
044import org.forgerock.audit.retention.RetentionPolicy;
045import org.forgerock.audit.retention.SizeBasedRetentionPolicy;
046import org.forgerock.audit.retention.TimeStampFileNamingPolicy;
047import org.forgerock.audit.rotation.FixedTimeRotationPolicy;
048import org.forgerock.audit.rotation.RotatableObject;
049import org.forgerock.audit.rotation.RotationContext;
050import org.forgerock.audit.rotation.RotationHooks;
051import org.forgerock.audit.rotation.RotationPolicy;
052import org.forgerock.audit.rotation.SizeBasedRotationPolicy;
053import org.forgerock.audit.rotation.TimeLimitRotationPolicy;
054import org.forgerock.util.annotations.VisibleForTesting;
055import org.forgerock.util.time.Duration;
056import org.joda.time.DateTime;
057import org.joda.time.DateTimeZone;
058import org.slf4j.Logger;
059import org.slf4j.LoggerFactory;
060
061/**
062 * Creates an {@link RotatableWriter} that supports file rotation and retention.
063 */
064public class RotatableWriter implements TextWriter, RotatableObject {
065
066    private static final Logger logger = LoggerFactory.getLogger(RotatableWriter.class);
067    private static final Duration ZERO = Duration.duration("zero");
068    private static final Duration FIVE_SECONDS = Duration.duration("5s");
069
070    private final List<RotationPolicy> rotationPolicies = new LinkedList<>();
071    private final List<RetentionPolicy> retentionPolicies = new LinkedList<>();
072    private final FileNamingPolicy fileNamingPolicy;
073    private ScheduledExecutorService rotator;
074    private DateTime lastRotationTime;
075    private final boolean rotationEnabled;
076    private final File file;
077    private RotationHooks rotationHooks = new RotationHooks.NoOpRotatationHooks();
078    private final AtomicBoolean isRotating = new AtomicBoolean(false);
079    /** The underlying output stream. */
080    private MeteredStream meteredStream;
081    /** The underlying buffered writer using the output stream. */
082    private BufferedWriter writer;
083    private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock();
084    private final RolloverLifecycleHook rolloverLifecycleHook;
085
086    /**
087     * Constructs a {@link RotatableWriter} given an initial file to manage rotation/retention, and
088     * a {@link FileBasedEventHandlerConfiguration}.
089     * @param file The initial file to manage rotation/retention.
090     * @param configuration The configuration of the rotation and retention policies.
091     * @param append Whether to append to the rotatable file or not.
092     * @throws IOException If a problem occurs.
093     */
094    public RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration,
095            final boolean append) throws IOException {
096        this(file, configuration, append, getTimeStampFileNamingPolicy(file, configuration));
097    }
098
099    /**
100     * Constructs a {@link RotatableWriter} given an initial file to manage rotation/retention, a
101     * a {@link FileBasedEventHandlerConfiguration} and a {@link RolloverLifecycleHook}.
102     *
103     * @param file The initial file to manage rotation/retention.
104     * @param configuration The configuration of the rotation and retention policies.
105     * @param append Whether to append to the rotatable file or not.
106     * @param rolloverLifecycleHook Hook to use before and after rotation/retention checks.
107     * @throws IOException If a problem occurs.
108     */
109    public RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration,
110            final boolean append, final RolloverLifecycleHook rolloverLifecycleHook) throws IOException {
111        this(file, configuration, append, getTimeStampFileNamingPolicy(file, configuration), rolloverLifecycleHook);
112    }
113
114    /**
115     * This constructor allows tests to set an alternative FileNamingPolicy as TimeStampFileNamingPolicy lists files
116     * for deletion by their last modified timestamps but these timestamps are only accurate to the nearest second.
117     */
118    @VisibleForTesting
119    RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration,
120                           final boolean append, final FileNamingPolicy fileNamingPolicy) throws IOException {
121        this(file, configuration, append, fileNamingPolicy, NOOP_ROLLOVER_LIFECYCLE_HOOK);
122    }
123
124    /** Constructor with all possible parameters. */
125    private RotatableWriter(final File file, final FileBasedEventHandlerConfiguration configuration,
126            final boolean append, final FileNamingPolicy fileNamingPolicy,
127            final RolloverLifecycleHook rolloverLifecycleHook) throws IOException {
128        this.file = file;
129        this.fileNamingPolicy = fileNamingPolicy;
130        this.rotationEnabled = configuration.getFileRotation().isRotationEnabled();
131        final long lastModified = file.lastModified();
132        this.lastRotationTime = lastModified > 0
133                ? new DateTime(file.lastModified(), DateTimeZone.UTC)
134                : DateTime.now(DateTimeZone.UTC);
135        this.rolloverLifecycleHook = rolloverLifecycleHook;
136        this.writer = constructWriter(file, append);
137        addRetentionPolicies(configuration.getFileRetention());
138        addRotationPolicies(configuration.getFileRotation());
139        scheduleRotationAndRetentionChecks(configuration);
140    }
141
142    private static TimeStampFileNamingPolicy getTimeStampFileNamingPolicy(final File file,
143            final FileBasedEventHandlerConfiguration configuration) {
144        return new TimeStampFileNamingPolicy(
145                file,
146                configuration.getFileRotation().getRotationFileSuffix(),
147                configuration.getFileRotation().getRotationFilePrefix());
148    }
149
150    /**
151     * Rotate the log file if any of the configured rotation policies determine that rotation is required.
152     *
153     * @throws IOException If unable to rotate the log file.
154     */
155    @Override
156    public void rotateIfNeeded() throws IOException {
157        if (!rotationEnabled || isRotating.get()) {
158            return;
159        }
160        readWriteLock.writeLock().lock();
161        try {
162            for (RotationPolicy rotationPolicy : rotationPolicies) {
163                if (rotationPolicy.shouldRotateFile(this)) {
164                    if (logger.isTraceEnabled()) {
165                        logger.trace("Must rotate: {}", file.getAbsolutePath());
166                    }
167                    isRotating.set(true);
168                    if (rotate()) {
169                        if (logger.isTraceEnabled()) {
170                            logger.trace("Finished rotation for: {}", file.getAbsolutePath());
171                        }
172                    }
173                    break;
174                }
175            }
176        } finally {
177            readWriteLock.writeLock().unlock();
178            isRotating.set(false);
179        }
180    }
181
182    /** Delete files if they need to be deleted as per enabled retention policies. */
183    private void deleteFilesIfNeeded() throws IOException {
184        readWriteLock.writeLock().lock();
185        try {
186            Set<File> filesToDelete = checkRetention(); // return the files to delete, but do not delete them
187            if (!filesToDelete.isEmpty()) {
188                deleteFiles(filesToDelete);
189            }
190        } finally {
191            readWriteLock.writeLock().unlock();
192        }
193    }
194
195    private boolean rotate() throws IOException {
196        boolean rotationHappened = false;
197        RotationContext context = new RotationContext();
198        context.setWriter(writer);
199        File currentFile = fileNamingPolicy.getInitialName();
200        context.setInitialFile(currentFile);
201        if (currentFile.exists()) {
202            File newFile = fileNamingPolicy.getNextName();
203            context.setNextFile(newFile);
204            rotationHooks.preRotationAction(context);
205            writer.close();
206            if (logger.isTraceEnabled()) {
207                logger.trace("Renaming {} to {}", currentFile.getAbsolutePath(), newFile.getAbsolutePath());
208            }
209            if (currentFile.renameTo(newFile)) {
210                rotationHappened = true;
211                if (currentFile.createNewFile()) {
212                    writer = constructWriter(currentFile, true);
213                    context.setWriter(writer);
214                    rotationHooks.postRotationAction(context);
215                } else {
216                    logger.error("Unable to resume writing to audit file {}; further events will not be logged",
217                            currentFile.toString());
218                }
219            } else {
220                logger.error("Unable to rename the audit file {}; further events will continue to be logged to "
221                        + "the current file", currentFile.toString());
222                writer = constructWriter(currentFile, true);
223            }
224            lastRotationTime = DateTime.now(DateTimeZone.UTC);
225        }
226        return rotationHappened;
227    }
228
229    private Set<File> checkRetention() throws IOException {
230        Set<File> filesToDelete = new HashSet<>();
231        for (RetentionPolicy retentionPolicy : retentionPolicies) {
232            filesToDelete.addAll(retentionPolicy.deleteFiles(fileNamingPolicy));
233        }
234        return filesToDelete;
235    }
236
237    private void deleteFiles(final Set<File> files) {
238        for (final File file : files) {
239            if (logger.isInfoEnabled()) {
240                logger.info("Deleting file {}", file.getAbsolutePath());
241            }
242            if (!file.delete()) {
243                if (logger.isWarnEnabled()) {
244                    logger.warn("Could not delete file {}", file.getAbsolutePath());
245                }
246            }
247        }
248    }
249
250    /**
251     * {@inheritDoc}
252     */
253    @Override
254    public long getBytesWritten() {
255        logger.trace("bytes written={}", meteredStream.getBytesWritten());
256        return meteredStream.getBytesWritten();
257    }
258
259    /**
260     * {@inheritDoc}
261     */
262    @Override
263    public DateTime getLastRotationTime() {
264        return lastRotationTime;
265    }
266
267    /**
268     * {@inheritDoc}
269     */
270    @Override
271    public void close() throws IOException {
272        if (rotator != null) {
273            boolean interrupted = false;
274            rotator.shutdown();
275            try {
276                while (!rotator.awaitTermination(500, MILLISECONDS)) {
277                    logger.debug("Waiting to terminate the rotator thread.");
278                }
279            } catch (InterruptedException ex) {
280                logger.error("Unable to terminate the rotator thread", ex);
281                interrupted = true;
282            } finally {
283                if (interrupted) {
284                    Thread.currentThread().interrupt();
285                }
286            }
287        }
288        writer.close();
289    }
290
291    @Override
292    public void shutdown() {
293        try {
294            close();
295        } catch (IOException e) {
296            logger.error("Error when performing shutdown", e);
297        }
298    }
299
300    /**
301     * {@inheritDoc}
302     */
303    @Override
304    public void registerRotationHooks(final RotationHooks rotationHooks) {
305        this.rotationHooks = rotationHooks;
306    }
307
308    @Override
309    public void write(String str) throws IOException {
310        ReadLock lock = readWriteLock.readLock();
311        try {
312            lock.lock();
313            logger.trace("Actually writing to file: {}", str);
314            writer.write(str);
315        } finally {
316            lock.unlock();
317        }
318        rotateIfNeeded();
319    }
320
321    /**
322     * Forces a rotation of the writer.
323     *
324     * @return {@code true} if rotation was done, {@code false} otherwise.
325     * @throws IOException
326     *          If an error occurs
327     */
328    public boolean forceRotation() throws IOException {
329        readWriteLock.writeLock().lock();
330        try {
331            isRotating.set(true);
332            return rotate();
333        } finally {
334            isRotating.set(false);
335            readWriteLock.writeLock().unlock();
336        }
337    }
338
339    @Override
340    public void flush() throws IOException {
341        writer.flush();
342    }
343
344    private BufferedWriter constructWriter(File csvFile, boolean append)
345            throws IOException {
346        FileOutputStream stream = new FileOutputStream(csvFile, append);
347        meteredStream = new MeteredStream(stream, file.length());
348        OutputStreamWriter osw = new OutputStreamWriter(meteredStream, StandardCharsets.UTF_8);
349        return new BufferedWriter(osw);
350    }
351
352    private void addRotationPolicies(final FileRotation fileRotation) {
353        // add SizeBasedRotationPolicy if a non zero size is supplied
354        final long maxFileSize = fileRotation.getMaxFileSize();
355        if (maxFileSize > 0) {
356            rotationPolicies.add(new SizeBasedRotationPolicy(maxFileSize));
357        }
358
359        // add FixedTimeRotationPolicy
360        final List<Duration> dailyRotationTimes = new LinkedList<>();
361        for (final String rotationTime : fileRotation.getRotationTimes()) {
362            Duration duration = parseDuration("rotation time", rotationTime, null);
363            if (duration != null && !duration.isUnlimited()) {
364                dailyRotationTimes.add(duration);
365            }
366        }
367        if (!dailyRotationTimes.isEmpty()) {
368            rotationPolicies.add(new FixedTimeRotationPolicy(dailyRotationTimes));
369        }
370
371        // add TimeLimitRotationPolicy if enabled
372        final Duration rotationInterval = parseDuration("rotation interval", fileRotation.getRotationInterval(), ZERO);
373        if (!(rotationInterval.isZero() || rotationInterval.isUnlimited())) {
374            rotationPolicies.add(new TimeLimitRotationPolicy(rotationInterval));
375        }
376    }
377
378    /**
379     * Schedule checks for rotations and retention policies.
380     * <p>
381     * The check interval is provided by the RotationRetentionCheckInterval property, which must have
382     * a non-zero value if at least one policy is enabled.
383     */
384    private void scheduleRotationAndRetentionChecks(FileBasedEventHandlerConfiguration configuration)
385            throws IOException {
386        final Duration rotationCheckInterval = parseDuration("rotation and retention check interval",
387                configuration.getRotationRetentionCheckInterval(), FIVE_SECONDS);
388
389        if (!rotationPolicies.isEmpty() || !retentionPolicies.isEmpty()) {
390            if (rotationCheckInterval.isUnlimited() || rotationCheckInterval.isZero()) {
391                throw new IOException("Rotation and retention check interval set to an invalid value: "
392                        + rotationCheckInterval);
393            }
394            rotator = Executors.newScheduledThreadPool(1);
395            rotator.scheduleAtFixedRate(
396                    new Runnable() {
397                        @Override
398                        public void run() {
399                            rolloverLifecycleHook.beforeRollingOver();
400                            try {
401                                try {
402                                    rotateIfNeeded();
403                                } catch (Exception e) {
404                                    logger.error("Failure when applying a rotation policy to file {}",
405                                            fileNamingPolicy.getInitialName(), e);
406                                }
407                                try {
408                                    deleteFilesIfNeeded();
409                                } catch (Exception e) {
410                                    logger.error("Failure when applying a retention policy to file {}",
411                                            fileNamingPolicy.getInitialName(), e);
412                                }
413                            } finally {
414                                rolloverLifecycleHook.afterRollingOver();
415                            }
416                        }
417                    },
418                    rotationCheckInterval.to(TimeUnit.MILLISECONDS),
419                    rotationCheckInterval.to(TimeUnit.MILLISECONDS),
420                    TimeUnit.MILLISECONDS);
421        }
422    }
423
424    private Duration parseDuration(String description, String duration, Duration defaultValue) {
425        try {
426            return Duration.duration(duration);
427        } catch (IllegalArgumentException e) {
428            logger.warn("Invalid {} value: '{}'", description, duration);
429            return defaultValue;
430        }
431    }
432
433    @VisibleForTesting
434    List<RotationPolicy> getRotationPolicies() {
435        return rotationPolicies;
436    }
437
438    private void addRetentionPolicies(final FileRetention fileRetention) {
439        // Add SizeBasedRetentionPolicy if the max number of files config value is more than 0
440        final int maxNumberOfHistoryFiles = fileRetention.getMaxNumberOfHistoryFiles();
441        if (maxNumberOfHistoryFiles > 0) {
442            retentionPolicies.add(new SizeBasedRetentionPolicy(maxNumberOfHistoryFiles));
443        }
444
445        // Add DiskSpaceUsedRetentionPolicy if config value > 0
446        final long maxDiskSpaceToUse = fileRetention.getMaxDiskSpaceToUse();
447        if (maxDiskSpaceToUse > 0) {
448            retentionPolicies.add(new DiskSpaceUsedRetentionPolicy(maxDiskSpaceToUse));
449        }
450
451        // Add FreeDiskSpaceRetentionPolicy if config value > 0
452        final long minimumFreeDiskSpace = fileRetention.getMinFreeSpaceRequired();
453        if (minimumFreeDiskSpace > 0) {
454            retentionPolicies.add(new FreeDiskSpaceRetentionPolicy(minimumFreeDiskSpace));
455        }
456    }
457
458    /**
459     * A RotationRetentionCheckHook that does nothing.
460     */
461    public static final RolloverLifecycleHook NOOP_ROLLOVER_LIFECYCLE_HOOK =
462        new RolloverLifecycleHook() {
463            @Override
464            public void beforeRollingOver() {
465                // nothing to do
466            }
467
468            @Override
469            public void afterRollingOver() {
470                // nothing to do
471            }
472    };
473
474    /**
475     * Callback hooks to allow custom action to be taken before and after the checks for rotation and
476     * retention is performed.
477     */
478    public interface RolloverLifecycleHook {
479
480        /**
481         * This method is called before the rotation and retention checks are done.
482         */
483        void beforeRollingOver();
484
485        /**
486         * This method is called after the rotation and retention checks are done.
487         */
488        void afterRollingOver();
489    }
490
491}