#!/bin/bash source "$( dirname $( readlink -f "$0" ) )/shflags.sh" # Filter options DEFINE_integer min_rating 3 "The min rating for files to be processed (0-5)." DEFINE_integer max_age_days -1 "The maximum age (since the creation date) of an image to be processed." DEFINE_string exiftool_filter "" "Only process images that satisfy the condition, which must be a string that can be evaluated by exiftool's -if statement." # Path options DEFINE_string photos_dir "" "The directory that contains the photos to be processed." DEFINE_string bestof_dir "" "The directory where the photos above --min_rating should be copied to. Path will be created if it does not exist." # Process options DEFINE_boolean resize false "Whether to resize the photos before copying to --resolution." DEFINE_string resize_to 1920 "The resolution to which to resize the photos if --resize is set. Can be anything that \\\`convert -resize\\\` accepts." DEFINE_boolean flatten false "Whether to flatten the directory structure to have all image files in 'best_of' directory." DEFINE_boolean force "false" "Whether to force reprocessing of all files instead of just new files." DEFINE_boolean prune "false" "Whether to remove files already existing in the bestof folder that do not meet the filter criteria." DEFINE_boolean use_hardlinks "false" "Whether to use hardlinks instead of copying. Cannot be combined with --resize." # Debug options DEFINE_boolean verbose false "Whether to print all actions performed." DEFINE_boolean dry_run false "Only print what would be done but do not do anything." DEFINE_boolean debug false "Whether to print bash debug output." set -euo pipefail EXCLUDE_MARKER=".bestof-exclude" PROCESSED_FILES_STORE=".bestof-processed" function error() { echo "$@" >&2 } function verbose() { if [[ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]]; then error "$@" fi } function paths_in_same_filesystem() { fs_id1=$( stat -c "%d" "$1" ) fs_id2=$( stat -c "%d" "$2" ) [[ "$fs_id1" = "$fs_id2" ]] } function validate_flags() { [[ ! -d "${FLAGS_photos_dir}" ]] && error "--photos_dir must be set and point to a valid directory." && return 1 [[ -z "${FLAGS_bestof_dir}" ]] && error "--bestof_dir must be set." && return 1 if [[ ${FLAGS_resize} -eq ${FLAGS_TRUE} ]]; then [[ -z "${FLAGS_resize_to}" ]] && error "--resize_to cannot be empty when --resize is given." && return 1 fi if [[ ${FLAGS_use_hardlinks} -eq ${FLAGS_TRUE} ]]; then if [[ ${FLAGS_resize} -eq ${FLAGS_TRUE} ]]; then error "--use_hardlinks and --resize are incompatible." return 1 fi if ! paths_in_same_filesystem "${FLAGS_photos_dir}" "${FLAGS_bestof_dir}"; then error "--use_hardlinks can only be used when --photos_dir and --bestof_dir are in the same filesystem." return 1 fi fi return 0 } function process_flags() { if [[ ${FLAGS_dry_run} -eq ${FLAGS_TRUE} ]]; then echo "Dry-run: Not doing anything..." fi if [[ ${FLAGS_debug} -eq ${FLAGS_TRUE} ]]; then set -x fi mkdir -p "${FLAGS_bestof_dir}" return 0 } function is_already_processed() { if [[ ${FLAGS_force} -eq ${FLAGS_FALSE} ]] && \ [[ -f "$PROCESSED_FILES_STORE_PATH" ]] && \ look -b "$photo_path" "$PROCESSED_FILES_STORE_PATH" &> /dev/null; then # Return that file does not need to be processed again. return 0 fi # Mark file as processed for next time. echo "$photo_path" >> "$PROCESSED_FILES_STORE_PATH" # Return that file should be processed. return 1 } function maybe_run() { if [[ ${FLAGS_dry_run} -eq ${FLAGS_FALSE} ]]; then # Run the command "$@" fi } function file_passes_filters() { local file_path file_path="$1" condition='$Rating >= '${FLAGS_min_rating} if [[ ${FLAGS_max_age_days} -gt -1 ]]; then condition="$condition and ( not ShiftTime(\$now,\"-${FLAGS_max_age_days} 0\") and \$CreateDate gt \$now )" fi if [[ -n "${FLAGS_exiftool_filter}" ]]; then condition="$condition and ${FLAGS_exiftool_filter}" fi # Use -TagDoesNotExist to suppress any output by exiftool as we only care about the return value. verbose "exiftool -if \"$condition\" -TagDoesNotExist -q -q -m \"$photo_path\"" exiftool -if "$condition" -TagDoesNotExist -q -q -m "$photo_path" } function main() { validate_flags process_flags PROCESSED_FILES_STORE_PATH="${FLAGS_bestof_dir}/$PROCESSED_FILES_STORE" [[ -f "$PROCESSED_FILES_STORE_PATH" ]] && LC_COLLATE=C sort -u -d -o "$PROCESSED_FILES_STORE_PATH" "$PROCESSED_FILES_STORE_PATH" # Exclude directories containing the .bestof-exclude marker. See https://unix.stackexchange.com/questions/468734/find-exclude-path-containing-specific-files for details. # Also exclude the bestof dir if it is part of the photos_dir total=$( find "${FLAGS_photos_dir}" \( -exec [ -f {}/.bestof-exclude ] \; -prune \) \ -o -path "${FLAGS_bestof_dir}" -prune \ -o -iname "*.jpg" -print | wc -l ) count=0 processed_cnt=0 while read photo_path; do count=$(( $count + 1 )); if [[ $(( $count % 10 )) -eq 0 ]] || [[ $count -eq $total ]]; then echo "$( date +%T ) Visited $count/$total image files. Images copied to best-of folder: $processed_cnt..." fi verbose "Processing file $photo_path." # Only record processed state when not doing a dry run. if [[ ${FLAGS_dry_run} -eq ${FLAGS_FALSE} ]] && is_already_processed; then verbose " Skipping file as it was already processed." continue fi rel_path="${photo_path#${FLAGS_photos_dir}}" if [[ ${FLAGS_flatten} -eq ${FLAGS_TRUE} ]]; then # Drop the complete path if we are flattening. rel_path=$( basename "$rel_path" ) fi bestof_path="${FLAGS_bestof_dir}/${rel_path}" # Delete file with the same name if it already exists. [[ -f "${bestof_path}" ]] && rm "${bestof_path}" if ! file_passes_filters "$photo_path"; then verbose " File does not pass filters." continue fi # Create all parent directories. mkdir -p "${bestof_path%/*}" if [[ ${FLAGS_resize} -eq ${FLAGS_TRUE} ]]; then verbose " Resizing file into target directory." maybe_run convert "${photo_path}" -resize "${FLAGS_resize_to}" "${bestof_path}" else if [[ ${FLAGS_use_hardlinks} -eq ${FLAGS_TRUE} ]]; then verbose " Hardlinking file into target directory." maybe_run ln "${photo_path}" "${bestof_path}" else verbose " Copying file into target directory." maybe_run cp "${photo_path}" "${bestof_path}" fi fi processed_cnt=$(( $processed_cnt + 1 )) done < <( find "${FLAGS_photos_dir}" \( -exec [ -f "{}/$EXCLUDE_MARKER" ] \; -prune \) \ -o -path "${FLAGS_bestof_dir}" -prune \ -o -iname "*.jpg" -print ) pruned_cnt=0 if [[ ${FLAGS_prune} -eq ${FLAGS_TRUE} ]]; then echo "Pruning filtered files in ${FLAGS_bestof_dir}..." while read photo_path; do echo "Processing file $photo_path." if ! file_passes_filters "$photo_path"; then verbose " Pruning file as it does not pass the filters." maybe_run rm "$photo_path" pruned_cnt=$(( $pruned_cnt + 1 )) fi done < <( find "${FLAGS_bestof_dir}" -iname "*.jpg" ) [[ $pruned_cnt -gt 0 ]] && echo "Pruned $pruned_cnt files from best-of directory." fi } # Parse the command-line FLAGS "$@" || exit $? eval set -- "${FLAGS_ARGV}" main "$@"