diff --git a/audio-manager.rkt b/audio-manager.rkt index ef103bf..0d09b38 100644 --- a/audio-manager.rkt +++ b/audio-manager.rkt @@ -1,18 +1,19 @@ #lang racket/base -(require racket/file) -(require racket/serialize) -(require racket/string) -(require "private/file-walker.rkt") -(require "private/util.rkt") -(require "private/mail.rkt") -(require "private/flac-handling.rkt") -(require "private/opus-handling.rkt") -(require "private/log.rkt") -(require "private/store.rkt") -(require racket-audio) -(require simple-log) -(require simple-ini) +(require racket/file + racket/serialize + racket/port + racket/string + "private/file-walker.rkt" + "private/util.rkt" + "private/mail.rkt" + "private/flac-handling.rkt" + "private/opus-handling.rkt" + "private/log.rkt" + "private/store.rkt" + racket-audio + simple-log + simple-ini) (provide audio-manager) @@ -44,7 +45,13 @@ (copied-files 0) (failed-converts 0) (failed-copies 0) + (files-in-target 0) + (dirs-in-target 0) + (last-dirs-in-target -1) (not-in-source 0) + (jobs 0) + (max-jobs (ini-get ini 'audio-manager 'max-jobs 6)) + (job-nr 0) (processed-kind (make-hash)) (base-rate (ini-get ini 'flac 'max-khz 48000)) (dirs-logged -1) @@ -113,6 +120,10 @@ (eq? s 'flac)) #f)) + (define (opus? info) + (and (eq? (hash-ref info 'type) 'file) + (eq? (hash-ref info 'ext) 'opus))) + (define (rep-id3 path) (set! report-flacs-with-id3 (string-append report-flacs-with-id3 @@ -222,6 +233,7 @@ ) (values base-path path info)) + ;; Converteer naar opus indien nodig en flac. (define (to-opus base-path path info) (when (and (needs-processing? path info) @@ -234,28 +246,49 @@ (set! processed-opus (+ processed-opus 1)) (info-am "Converting flac to opus: ~a" opus-file) (rep-opus - (format "Converting flac to opus: ~a" (basename opus-file))) - (with-handlers ([exn? (λ (e) - (set! failed-converts (+ failed-converts 1)) - (err-am "Conversion of ~a: ~a" path e))]) - (unless (directory-exists? (basedir opus-file)) - (make-directory* (basedir opus-file))) - (if (convert-to-opus path opus-file ini) + (format "Converting flac to opus in thread: ~a" (basename opus-file))) + + + (let loop () + (if (= jobs max-jobs) (begin - (set! converted-files (+ converted-files 1)) - (info-am " Converted ~a" converted-files) - ) + (sleep 0.1) + (loop)) (begin - (info-am " CONVERSION PROBLEM") - (rep-opus " Conversion failed!") - (set! failed-converts (+ failed-converts 1)) + (set! job-nr (+ job-nr 1)) + (set! jobs (+ jobs 1)) + (let ((jobnr job-nr)) + (thread + (λ () + (with-handlers ([exn? (λ (e) + (set! failed-converts (+ failed-converts 1)) + (err-am "Conversion of ~a: ~a" path e))]) + (unless (directory-exists? (basedir opus-file)) + (make-directory* (basedir opus-file))) + (info-am " Starting conversion job ~a" jobnr) + (if (convert-to-opus path opus-file ini) + (begin + (set! converted-files (+ converted-files 1)) + (info-am " Converted ~a, job-nr = ~a" converted-files jobnr) + ) + (begin + (info-am " CONVERSION PROBLEM, job-nr = ~a" jobnr) + (rep-opus " Conversion failed!") + (set! failed-converts (+ failed-converts 1)) + ) + ) + (set! jobs (- jobs 1)) + ) + ) + #:pool 'own + ) + ) ) ) ) ) ) - ) - ) + )) (values base-path path info)) ;; Kopieer bestanden rechtstreeks indien nodig en geen flac @@ -300,10 +333,27 @@ (values base-path path info)) (define (target-tree-cleaner base-path path info) + (if (eq? (hash-ref info 'type #f) 'dir) + (set! dirs-in-target (+ dirs-in-target 1)) + (set! files-in-target (+ files-in-target 1))) + + (when (or (and + (= (remainder dirs-in-target 100) 0) + (> (quotient dirs-in-target 100) last-dirs-in-target)) + (= (remainder files-in-target 1000) 0)) + (set! last-dirs-in-target (quotient dirs-in-target 100)) + (info-am "Files in target checked: ~a, dirs: ~a" files-in-target dirs-in-target) + (sync-log-am) + ) (let ((normalized-sub-path (hash-ref info 'path))) + (when (opus? info) + (set! normalized-sub-path (path-replace-extension normalized-sub-path #".flac"))) (unless (store-exists? file-db-store normalized-sub-path) - (set! not-in-source (+ not-in-source 1)))) - (values base-path path info)) + (info-am "Not in source: ~a" normalized-sub-path) + (delete-directory/files path #:must-exist? #f) + (sync-log-am) + (set! not-in-source (+ not-in-source 1))) + (values base-path path info))) (define (log-processed-file base-path path info) (dbg-am "processing ~a" path) @@ -323,8 +373,8 @@ (hash-set! processed-kind ext n))) ) - (when (or (= (remainder processed-dirs 10) 0) - (= (remainder processed-files 250) 0)) + (when (or (= (remainder processed-dirs 100) 0) + (= (remainder processed-files 1000) 0)) (unless (= dirs-logged processed-dirs) (set! dirs-logged processed-dirs) (info-am (format "processed dirs: ~a, files: ~a, kinds: ~a, unknown kinds: ~a" @@ -340,6 +390,7 @@ (map (λ (ext) (symbol->string ext)) unknown-exts) ", ") )) + (sync-log-am) )) (when (flac-id3? info) @@ -375,6 +426,8 @@ (format "Copied files : ~a" copied-files) (format "Failed convert : ~a" failed-converts) (format "Failed copies : ~a" failed-copies) + (format "Files in target: ~a" files-in-target) + (format "Dirs in target : ~a" dirs-in-target) (format "Not in source : ~a" not-in-source) )) (subj (format "Audio manager report d.d. ~a" (date->yyyy-mm-dd (now)))) @@ -409,6 +462,14 @@ (let-values (((base-path path info) (fw))) (if (eq? info #f) (begin + ; Wait for all conversions to finish + (let loop () + (if (= jobs 0) + #t + (begin + (sleep 0.1) + (loop)))) + ; Process Rest. (info-am "Finished walking music library") ;; Cleanup files in target, not in source (info-am "Checking files in target, not in source") @@ -416,7 +477,9 @@ (let loop1 () (let-values (((base-path path info) (fwt))) (if (eq? info #f) - (info-am "Target check finished") + (begin + (info-am "Target check finished, not in source: ~a" not-in-source) + (info-am "Files in target checked: ~a" files-in-target)) (loop1))))) (info-am "Closing store") (store-close file-db-store) diff --git a/private/file-walker.rkt b/private/file-walker.rkt index 740c185..19ba76d 100644 --- a/private/file-walker.rkt +++ b/private/file-walker.rkt @@ -130,7 +130,8 @@ (define (get-deleted) (let ((count (store-count db-store)) - (k 0)) + (k 0) + (last-perc 0)) (let ((keys (store-keys db-store))) (map (λ (key) (let ((info (store-ref db-store key))) @@ -139,7 +140,8 @@ (filter (λ (key) (set! k (+ k 1)) (let ((perc (quotient (* k 100) count))) - (when (= (remainder perc 10) 0) + (when (and (= (remainder perc 10) 0) (> perc last-perc)) + (set! last-perc perc) (info-am "get-deleteed: ~a%" perc))) (let ((v (store-ref db-store key))) (eq? (hash-ref v 'file-db) 'unknown))) @@ -147,14 +149,20 @@ ) (let* ((walker-count 0) + (last-commit-s (current-seconds)) (fw (make-file-walker base-path "*" (λ (base-path path info) - (cond - ((= walker-count 0) - (store-begin db-store)) - ((= (remainder walker-count 250) 0) - (store-commit db-store) - (store-begin db-store)) + (let ((tm (current-seconds))) + (cond + ((= walker-count 0) + (store-begin db-store)) + ((or (= (remainder walker-count 250) 0) + (> (- tm last-commit-s) 5)) + (store-commit db-store) + (store-begin db-store) + (set! last-commit-s tm) + ) + ) ) (set! walker-count (+ walker-count 1)) (let-values (((base-path* path* info*) (file-admin base-path path info))) diff --git a/private/store.rkt b/private/store.rkt index 24ac6c2..1eb442d 100644 --- a/private/store.rkt +++ b/private/store.rkt @@ -19,6 +19,9 @@ store-commit ) +(define (cvtkey k) + (string-downcase (format "~a" k))) + (define store-kind 'hash) (define (store-config! #:kind [kind 'hash]) @@ -48,34 +51,38 @@ (serialize st) file #:exists 'replace)) (ks-close (cadr st)))) -(define (store-ref st key . val) - (if (eq? (car st) 'hash) - (if (null? val) - (hash-ref (cadr st) key) - (hash-ref (cadr st) key (car val))) - (let ((v (if (null? val) - (ks-get (cadr st) key) +(define (store-ref st key* . val) + (let ((key (cvtkey key*))) + (if (eq? (car st) 'hash) + (if (null? val) + (hash-ref (cadr st) key) + (hash-ref (cadr st) key (car val))) + (let ((v (if (null? val) + (ks-get (cadr st) key) (ks-get (cadr st) key (car val))))) (if (eq? v 'ks-nil) (error "No such key in storage") v)) - ) - ) + ) + )) -(define (store-set! st key val) - (if (eq? (car st) 'hash) - (hash-set! (cadr st) key val) - (ks-set! (cadr st) key val))) +(define (store-set! st key* val) + (let ((key (cvtkey key*))) + (if (eq? (car st) 'hash) + (hash-set! (cadr st) key val) + (ks-set! (cadr st) key val)))) -(define (store-remove! st key) - (if (eq? (car st) 'hash) - (hash-remove! (cadr st) key) - (ks-drop! (cadr st) key))) +(define (store-remove! st key*) + (let ((key (cvtkey key*))) + (if (eq? (car st) 'hash) + (hash-remove! (cadr st) key) + (ks-drop! (cadr st) key)))) -(define (store-exists? st key) - (if (eq? (car st) 'hash) - (hash-has-key? (cadr st) key) - (ks-exists? (cadr st) key))) +(define (store-exists? st key*) + (let ((key (cvtkey key*))) + (if (eq? (car st) 'hash) + (hash-has-key? (cadr st) key) + (ks-exists? (cadr st) key)))) (define (store-keys st) (if (eq? (car st) 'hash)