diff --git a/README.md b/README.md index 2fa7443..25cf0ba 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,22 @@ # audio-library-manager -If you have a directory with, e.g. /music with a library containing flac music tracks, this will resample the files to at most 48.000Hz (get rid of the 96Khz/192Khz files that one will never hear the difference to 48Khz). Also there's a flac -> opus conversion path (to e.g. /music-opus) to make a portable library. It will keep the information about the files in /music/.music-info.db. \ No newline at end of file +First setup for `flac-48khz-manager.rkt`. + +The command keeps a FLAC directory tree at a maximum sample rate of 48 kHz. +Files above the configured threshold are converted in place through a Racket place. +The conversion path uses `racket-audio/audio-encoder` dynamically, so the package can still compile on systems where the native audio libraries are not installed yet. + +Default files below the FLAC root: + +- `.music-info.db`: keystore state database +- `.flac-48khz-manager.ini`: configuration +- `.flac-48khz-manager.log`: log file + +Run: + +```sh +racket flac-48khz-manager.rkt /path/to/flac-tree +``` + +Important configuration keys are created automatically in `.flac-48khz-manager.ini`. +SMTP reports are HTML and only summarize counters and errors. diff --git a/flac-48khz-manager.rkt b/flac-48khz-manager.rkt new file mode 100644 index 0000000..7918a69 --- /dev/null +++ b/flac-48khz-manager.rkt @@ -0,0 +1,125 @@ +#lang racket/base + +(require racket/cmdline + racket/file + racket/list + racket/path + "private/audio.rkt" + "private/config.rkt" + "private/convert-place.rkt" + "private/hash.rkt" + "private/log.rkt" + "private/mail.rkt" + "private/report.rkt" + "private/scan.rkt" + "private/state.rkt" + "private/util.rkt") + +(provide manage-flac-tree + summary->lines) + +(define (file-info path digest) + (list (cons 'digest digest) + (cons 'size (file-size path)) + (cons 'mtime (file-or-directory-modify-seconds path)))) + +(define (same-file-state? old digest) + (and old + (equal? (alist-ref/default old 'digest #f) digest) + (not (equal? (alist-ref/default old 'status #f) 'error)))) + +(define (process-one-file ks config path relpath inspect-flac-proc convert-proc summary errors) + (define digest (file-digest path (manager-config-hash-algorithm config))) + (define old (state-get-file ks relpath #f)) + (cond [(same-file-state? old digest) + (info-alm "unchanged: ~a" relpath) + (values (summary-inc (summary-inc summary 'seen) 'unchanged) errors)] + [else + (define summary1 (summary-inc (summary-inc summary 'seen) 'processed)) + (define summary2 (if old (summary-inc summary1 'changed) (summary-inc summary1 'new))) + (with-handlers ([exn:fail? + (lambda (e) + (err-alm "error for ~a: ~a" relpath (exn-message e)) + (state-set-file! ks relpath + (append (file-info path digest) + (list (cons 'status 'error) + (cons 'message (exn-message e))))) + (values (summary-inc summary2 'errors) + (cons (list (cons 'file relpath) + (cons 'message (exn-message e))) + errors)))]) + (define sample-rate (inspect-flac-proc path)) + (cond [(> sample-rate (manager-config-max-sample-rate config)) + (if (manager-config-dry-run? config) + (begin + (warn-alm "dry-run: would convert ~a from ~a Hz to ~a Hz" relpath sample-rate (manager-config-max-sample-rate config)) + (state-set-file! ks relpath + (append (file-info path digest) + (list (cons 'status 'dry-run) + (cons 'sample-rate sample-rate)))) + (values (summary-inc (summary-inc summary2 'skipped) 'dry-run) errors)) + (begin + (info-alm "converting ~a from ~a Hz to ~a Hz" relpath sample-rate (manager-config-max-sample-rate config)) + (let* ((result (convert-proc path + (manager-config-max-sample-rate config) + (manager-config-compression-level config))) + (new-digest (file-digest path (manager-config-hash-algorithm config))) + (new-sample-rate (inspect-flac-proc path))) + (state-set-file! ks relpath + (append (file-info path new-digest) + (list (cons 'status 'converted) + (cons 'old-sample-rate sample-rate) + (cons 'sample-rate new-sample-rate) + (cons 'encoder-result result)))) + (values (summary-inc summary2 'converted) errors))))] + [else + (info-alm "ok: ~a (~a Hz)" relpath sample-rate) + (state-set-file! ks relpath + (append (file-info path digest) + (list (cons 'status 'ok) + (cons 'sample-rate sample-rate)))) + (values (summary-inc summary2 'ok) errors)]))])) + +(define (drop-removed! ks current-relpaths summary) + (define current (for/hash ([r (in-list current-relpaths)]) (values r #t))) + (for/fold ([s summary]) ([old-rel (in-list (state-known-relpaths ks))]) + (if (hash-ref current old-rel #f) + s + (begin + (info-alm "removed from state: ~a" old-rel) + (state-drop-file! ks old-rel) + (summary-inc s 'removed))))) + +(define (manage-flac-tree base-directory + #:inspect-flac-proc [inspect-flac-proc inspect-flac-sample-rate] + #:convert-proc [convert-proc convert-flac-to-target-in-place]) + (define base-dir (simple-form-path base-directory)) + (unless (directory-exists? base-dir) + (raise-argument-error 'manage-flac-tree "existing directory" base-directory)) + (define config (load-manager-config base-dir)) + (setup-logging! (manager-config-log-file config) (manager-config-display-log? config)) + (info-alm "base directory: ~a" base-dir) + (info-alm "state file: ~a" (manager-config-state-file config)) + (info-alm "ini file: ~a" (manager-config-ini-file config)) + (define ks (open-manager-state (manager-config-state-file config))) + (define files (find-flac-files base-dir)) + (define relpaths (map (lambda (p) (relpath-string base-dir p)) files)) + (define summary0 (drop-removed! ks relpaths (make-empty-summary))) + (define-values (summary errors) + (for/fold ([summary summary0] [errors '()]) ([p (in-list files)] [rel (in-list relpaths)]) + (process-one-file ks config p rel inspect-flac-proc convert-proc summary errors))) + (define errors* (reverse errors)) + (with-handlers ([exn:fail? (lambda (e) (err-alm "mail report failed: ~a" (exn-message e)) (void))]) + (maybe-send-report-mail config summary errors*)) + (for ([line (in-list (summary->lines summary))]) (info-alm "summary: ~a" line)) + summary) + +(module+ main + (define base-dir #f) + (command-line + #:program "flac-48khz-manager.rkt" + #:args (base-directory) + (set! base-dir base-directory)) + (define summary (manage-flac-tree base-dir)) + (for ([line (in-list (summary->lines summary))]) + (displayln line))) diff --git a/info.rkt b/info.rkt new file mode 100644 index 0000000..71aa3cc --- /dev/null +++ b/info.rkt @@ -0,0 +1,16 @@ +#lang info + +(define collection "audio-library-manager") +(define deps '("base" + "db-lib" + "openssl" + "keystore" + "simple-ini" + "simple-log" + "smtp" + "racket-sprintf")) +(define scribblings '(("scribblings/audio-library-manager.scrbl" ()))) +(define build-deps '("rackunit-lib" "scribble-lib" "racket-doc")) +(define pkg-desc "Audio library maintenance tools for FLAC and Opus trees") +(define version "0.1") +(define pkg-authors '(hans-dijkema)) diff --git a/main.rkt b/main.rkt new file mode 100644 index 0000000..f742000 --- /dev/null +++ b/main.rkt @@ -0,0 +1,6 @@ +#lang racket/base + +(require "flac-48khz-manager.rkt") + +(provide manage-flac-tree + summary->lines) diff --git a/private/audio.rkt b/private/audio.rkt new file mode 100644 index 0000000..d55972c --- /dev/null +++ b/private/audio.rkt @@ -0,0 +1,16 @@ +#lang racket/base + +(provide inspect-flac-sample-rate) + +(define (inspect-flac-sample-rate path) + (define call-with-id3-tags (dynamic-require 'racket-audio/taglib 'call-with-id3-tags)) + (define tags-valid? (dynamic-require 'racket-audio/taglib 'tags-valid?)) + (define tags-sample-rate (dynamic-require 'racket-audio/taglib 'tags-sample-rate)) + (define sr + (call-with-id3-tags path + (lambda (tags) + (and (tags-valid? tags) (tags-sample-rate tags))) + #:mode 'read)) + (unless (and (integer? sr) (positive? sr)) + (error 'inspect-flac-sample-rate "cannot determine sample rate for ~a" path)) + sr) diff --git a/private/config.rkt b/private/config.rkt new file mode 100644 index 0000000..aef0c82 --- /dev/null +++ b/private/config.rkt @@ -0,0 +1,105 @@ +#lang racket/base + +(require racket/file + racket/path + simple-ini + "util.rkt") + +(provide manager-config? + manager-config-base-dir + manager-config-ini-file + manager-config-state-file + manager-config-log-file + manager-config-max-sample-rate + manager-config-hash-algorithm + manager-config-dry-run? + manager-config-display-log? + manager-config-compression-level + manager-config-mail-enabled? + manager-config-mail-send-on-success? + manager-config-mail-send-on-error? + manager-config-mail-host + manager-config-mail-port + manager-config-mail-tls? + manager-config-mail-username + manager-config-mail-password + manager-config-mail-from + manager-config-mail-to + manager-config-mail-cc + manager-config-mail-bcc + manager-config-mail-subject-prefix + load-manager-config + ensure-default-config!) + +(struct manager-config + (base-dir ini-file state-file log-file max-sample-rate hash-algorithm dry-run? + display-log? compression-level mail-enabled? mail-send-on-success? + mail-send-on-error? mail-host mail-port mail-tls? mail-username + mail-password mail-from mail-to mail-cc mail-bcc mail-subject-prefix) + #:transparent) + +(define (default-log-file base-dir) + (build-path base-dir ".flac-48khz-manager.log")) + +(define (default-ini base-dir) + (build-path base-dir ".flac-48khz-manager.ini")) + +(define (default-state base-dir) + (build-path base-dir ".music-info.db")) + +(define (ensure-default-config! ini-file) + (unless (file-exists? ini-file) + (define ini (make-ini)) + (ini-set! ini 'manager 'max-sample-rate 48000) + (ini-set! ini 'manager 'hash-algorithm "sha256") + (ini-set! ini 'manager 'dry-run #f) + (ini-set! ini 'manager 'display-log #t) + (ini-set! ini 'manager 'log-file ".flac-48khz-manager.log") + (ini-set! ini 'manager 'compression-level 5) + (ini-set! ini 'mail 'enabled #f) + (ini-set! ini 'mail 'send-on-success #f) + (ini-set! ini 'mail 'send-on-error #t) + (ini-set! ini 'mail 'host "") + (ini-set! ini 'mail 'port 25) + (ini-set! ini 'mail 'tls #f) + (ini-set! ini 'mail 'username "") + (ini-set! ini 'mail 'password "") + (ini-set! ini 'mail 'from "") + (ini-set! ini 'mail 'to "") + (ini-set! ini 'mail 'cc "") + (ini-set! ini 'mail 'bcc "") + (ini-set! ini 'mail 'subject-prefix "[flac-48khz-manager]") + (ini->file ini ini-file))) + +(define (resolve-log-file base-dir v) + (define p (string-value v ".flac-48khz-manager.log")) + (cond [(path-string? p) + (define bp (string->path p)) + (if (absolute-path? bp) bp (build-path base-dir bp))] + [else (default-log-file base-dir)])) + +(define (load-manager-config base-dir*) + (define base-dir (simple-form-path base-dir*)) + (define ini-file (default-ini base-dir)) + (ensure-default-config! ini-file) + (define ini (file->ini ini-file)) + (define log-file (resolve-log-file base-dir (ini-get ini 'manager 'log-file ".flac-48khz-manager.log"))) + (manager-config base-dir ini-file (default-state base-dir) log-file + (int-value (ini-get ini 'manager 'max-sample-rate 48000) 48000) + (string-downcase (string-value (ini-get ini 'manager 'hash-algorithm "sha256") "sha256")) + (bool-value (ini-get ini 'manager 'dry-run #f) #f) + (bool-value (ini-get ini 'manager 'display-log #t) #t) + (int-value (ini-get ini 'manager 'compression-level 5) 5) + (bool-value (ini-get ini 'mail 'enabled #f) #f) + (bool-value (ini-get ini 'mail 'send-on-success #f) #f) + (bool-value (ini-get ini 'mail 'send-on-error #t) #t) + (string-value (ini-get ini 'mail 'host "") "") + (int-value (ini-get ini 'mail 'port 25) 25) + (bool-value (ini-get ini 'mail 'tls #f) #f) + (string-value (ini-get ini 'mail 'username "") "") + (string-value (ini-get ini 'mail 'password "") "") + (string-value (ini-get ini 'mail 'from "") "") + (split-addresses (ini-get ini 'mail 'to "")) + (split-addresses (ini-get ini 'mail 'cc "")) + (split-addresses (ini-get ini 'mail 'bcc "")) + (string-value (ini-get ini 'mail 'subject-prefix "[flac-48khz-manager]") "[flac-48khz-manager]"))) diff --git a/private/convert-place.rkt b/private/convert-place.rkt new file mode 100644 index 0000000..94c6103 --- /dev/null +++ b/private/convert-place.rkt @@ -0,0 +1,46 @@ +#lang racket/base + +(require racket/file + racket/path + racket/place + "util.rkt") + +(provide convert-flac-to-target-in-place) + +(define (temp-output-path input-path) + (define-values (base name dir?) (split-path input-path)) + (define name-str (path->string name)) + (build-path base (format ".~a.tmp-~a.flac" name-str (current-inexact-milliseconds)))) + +(define (settings->alist max-sample-rate compression-level) + (list (cons 'target-sample-rate max-sample-rate) + (cons 'compression-level compression-level))) + +(define (convert-flac-to-target-in-place input-path max-sample-rate compression-level) + (define tmp-path (temp-output-path input-path)) + (define worker + (place ch + (define msg (place-channel-get ch)) + (define in-file (list-ref msg 0)) + (define out-file (list-ref msg 1)) + (define settings (list-ref msg 2)) + (with-handlers ([exn:fail? + (lambda (e) + (place-channel-put ch (list 'error (exn-message e))))]) + (define audio-encode (dynamic-require 'racket-audio/audio-encoder 'audio-encode)) + (define result (audio-encode in-file out-file (make-immutable-hash settings) + #:encoder 'flac + #:copy-tags? #t)) + (place-channel-put ch (list 'ok result))))) + (place-channel-put worker (list (path->string input-path) + (path->string tmp-path) + (settings->alist max-sample-rate compression-level))) + (define response (place-channel-get worker)) + (cond [(and (pair? response) (eq? (car response) 'ok)) + (rename-file-or-directory tmp-path input-path #t) + (cadr response)] + [else + (when (file-exists? tmp-path) (delete-file tmp-path)) + (error 'convert-flac-to-target-in-place "conversion failed for ~a: ~a" + input-path + (if (and (pair? response) (pair? (cdr response))) (cadr response) response))])) diff --git a/private/hash.rkt b/private/hash.rkt new file mode 100644 index 0000000..586eda8 --- /dev/null +++ b/private/hash.rkt @@ -0,0 +1,20 @@ +#lang racket/base + +(require openssl/sha1 + openssl/md5) + +(provide file-digest) + +(define (digest->string v) + (cond [(bytes? v) (bytes->hex-string v)] + [(string? v) v] + [else (format "~a" v)])) + +(define (file-digest path [algorithm "sha256"]) + (define alg (string-downcase (format "~a" algorithm))) + (call-with-input-file path + (lambda (in) + (cond [(member alg '("sha256" "sha-256" "sha2")) (digest->string (sha256-bytes in))] + [(member alg '("md5")) (digest->string (md5 in))] + [else (error 'file-digest "unsupported digest algorithm: ~a" algorithm)])) + #:mode 'binary)) diff --git a/private/log.rkt b/private/log.rkt new file mode 100644 index 0000000..e3e6c53 --- /dev/null +++ b/private/log.rkt @@ -0,0 +1,22 @@ +#lang racket/base + +(require racket/file + simple-log) + +(provide setup-logging! + dbg-alm + info-alm + warn-alm + err-alm + fatal-alm + sync-log-alm) + +(sl-def-log audio-library-manager alm) + +(define (setup-logging! log-file display-log?) + (make-directory* (let-values ([(base name dir?) (split-path log-file)]) base)) + (if display-log? + (sl-log-to-file&display log-file) + (sl-log-to-file log-file)) + (sl-set-log-level 'debug) + (info-alm "logging initialized: ~a" log-file)) diff --git a/private/mail.rkt b/private/mail.rkt new file mode 100644 index 0000000..2e76f4e --- /dev/null +++ b/private/mail.rkt @@ -0,0 +1,33 @@ +#lang racket/base + +(require smtp + "config.rkt" + "report.rkt") + +(provide maybe-send-report-mail) + +(define (maybe-send-report-mail config summary errors) + (define has-errors? (positive? (summary-ref summary 'errors 0))) + (define should-send? + (and (manager-config-mail-enabled? config) + (not (null? (manager-config-mail-to config))) + (or (and has-errors? (manager-config-mail-send-on-error? config)) + (and (not has-errors?) (manager-config-mail-send-on-success? config))))) + (when should-send? + (define subject (format "~a FLAC 48 kHz manager: ~a error(s), ~a converted" + (manager-config-mail-subject-prefix config) + (summary-ref summary 'errors 0) + (summary-ref summary 'converted 0))) + (define body (html-report subject summary errors)) + (define mail (make-mail subject body + #:from (manager-config-mail-from config) + #:to (manager-config-mail-to config) + #:cc (manager-config-mail-cc config) + #:bcc (manager-config-mail-bcc config) + #:body-content-type "text/html")) + (send-smtp-mail mail + #:host (manager-config-mail-host config) + #:port (manager-config-mail-port config) + #:tls-encode (manager-config-mail-tls? config) + #:username (manager-config-mail-username config) + #:password (manager-config-mail-password config)))) diff --git a/private/report.rkt b/private/report.rkt new file mode 100644 index 0000000..ae4d4dd --- /dev/null +++ b/private/report.rkt @@ -0,0 +1,57 @@ +#lang racket/base + +(require racket/list + racket/string + "util.rkt") + +(provide make-empty-summary + summary-inc + summary-set + summary-ref + summary->lines + html-report) + +(define summary-keys '(seen processed new changed unchanged ok converted removed skipped errors dry-run)) + +(define (make-empty-summary) + (append (for/list ([k (in-list summary-keys)]) (cons k 0)) + (list (cons 'error-list '())))) + +(define (summary-ref summary key [default 0]) + (alist-ref/default summary key default)) + +(define (summary-set summary key value) + (alist-set summary key value)) + +(define (summary-inc summary key [amount 1]) + (summary-set summary key (+ (summary-ref summary key 0) amount))) + +(define (summary->lines summary) + (for/list ([k (in-list summary-keys)]) + (format "~a: ~a" k (summary-ref summary k 0)))) + +(define (html-escape s) + (define x (format "~a" s)) + (define y (regexp-replace* #rx"&" x "&")) + (define z (regexp-replace* #rx"<" y "<")) + (regexp-replace* #rx">" z ">")) + +(define (html-report title summary errors) + (define rows + (apply string-append + (for/list ([k (in-list summary-keys)]) + (format "
No errors were reported.
" + (string-append + "| File | Error |
|---|---|
| ~a | ~a |