diff --git a/README.md b/README.md index cc9b4d6..a452fd4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ * Scans with RecursiveDirectoryIterator * Uses native macOS methods and CLI tools * Caches assets for re-scans -* Identifies the most useful information about a file +* Somewhat multithreaded +* Aggregates and summarizes metadata * Tested with large filesystems ## Screenshots diff --git a/Yuba.php b/Yuba.php index fca538f..8bf44e5 100755 --- a/Yuba.php +++ b/Yuba.php @@ -7,7 +7,6 @@ $version = file_get_contents(__DIR__."/version.txt"); ini_set('memory_limit', '10240M'); date_default_timezone_set("America/Los_Angeles"); -$mytime = time(); // Includes & Prefs ////////////////////////////////////////// @@ -24,9 +23,12 @@ $p = unserialize(file_get_contents($prefs_file)); require("functions.php"); require("filetypes.php"); -$wopt_steps = 12; +// Manual prefs + +$wopt_steps = 12; // total number of steps $wopt_currstep = 1; -$wopt_clear = 0; +$wopt_clear = 0; // set to 0 for debug +$wopt_parallel = 1; // use parallel? $parser = new plistParser(); @@ -60,7 +62,11 @@ if (@$argv[1] == "Check for Updates...") { // Path & application variables ////////////////////////////////////////// -$stamp = date("Y-m-d_H-i-s", time()); +$mytime = time(); +$tmpdir = "/tmp/yuba/".$mytime; +if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); } + +$stamp = date("Y-m-d_H-i-s", $mytime); if (!isset($argv[1])) { echo "No input"; die; } $zpath = realpath(@$argv[1]); @@ -560,148 +566,45 @@ foreach ($fx as $array) { } echo ProgressBar::finish($wopt_clear); -// Thumbnails +// Thumbs ////////////////////////////////////////// if ($p['thumbs']) { - echo ProgressBar::start(count($fx),"Generating thumbnails (".stepString().")"); + echo ProgressBar::start($passed_file,"Creating thumb batch (".stepString().")",1); + + $batchfile = $tmpdir."/_batch_thumbs.sh"; + $helper = realpath("helper.php"); - foreach ($fx as $array) { - + foreach ($fx as $count => $array) { $fid = $array[0]; $pathname = $array[1]; - $ext = pathinfo($pathname,PATHINFO_EXTENSION); - $tpath = $bpath."/thumbs/".substr($fid, 0, 2); - if (!is_dir($tpath)) { mkdir($tpath); } - - //$tfile = $tpath."/".$fid."_".$p['thumb_size'].".jpg"; - //$tpfile = $tpath."/".$fid."_".$p['thumb_size'].".png"; - - $tfile = $tpath."/".$fid.".jpg"; - $tpfile = $tpath."/".$fid.".png"; - - if (count($p['t_skip']) && in_array($ext, $p['t_skip'])) { - echo ProgressBar::next("Skipping ".shortlabel($pathname)); - continue; - } - - // check for rebuild mode - if ($p['thumbs'] == 2) { - $update_rowid = $dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()[0]; - } else { - if (is_file($tfile) | is_file($tpfile)) { - // faster to check for a file first - echo ProgressBar::next("Thumb file found for ".shortlabel($pathname)); - continue; - } elseif ($dbp->query("SELECT EXISTS(SELECT 1 FROM thumbs WHERE fid='".$fid."')")->fetch()[0]) { - // if no thumb file, then poll database for prior attempt - echo ProgressBar::next("Thumb record found for ".shortlabel($pathname)); - continue; - } else { - $update_rowid = false; - } - } - - if ($update_rowid) { - echo ProgressBar::next("Updating thumb for ".shortlabel($pathname)); - $stmt = $dbp->prepare("UPDATE thumbs SET fid = :fid, created = :created, relative_path = :relative_path, width = :width, height = :height, tool = :tool WHERE rowid = ".$update_rowid); - } else { - echo ProgressBar::next("Generating thumb for ".shortlabel($pathname)); - $stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)"); - } - - $shellpath = escapeshellarg($pathname); - $tmp_path = escapeshellarg("/tmp/".basename($pathname).".png"); // qlmanage workaround - - $fmt['sips'] = $tfile; - $fmt['sox'] = $tpfile; - $fmt['ffmpeg'] = $tfile; - $fmt['ql-thumbnail'] = $tfile; - $fmt['qltool'] = $tpfile; - $fmt['qlmanage'] = $tpfile; - $fmt['vips'] = $tfile; - - $cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tpfile; - $cmd['sips'] = "sips -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; - $cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile; - $cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8"; - //$cmd['qltool'] = $bin_qltool." di ".$shellpath." ".$p['thumb_size']." ".$p['thumb_size']." | base64 --decode | ".$bin_convert." - -scale 50% -strip -trim +repage -define png:compression-level=9 ".$tpfile; - $cmd['qltool'] = $bin_qltool." di ".$shellpath." ".$p['thumb_size']." ".$p['thumb_size']." | base64 --decode | ".$bin_convert." - -scale 50% -strip -define png:compression-level=9 ".$tpfile; - // qltool thumbs are double size with half the pixels - // repage crops thumbs to edge (match sips/ffmpeg thumbs) - $cmd['qlmanage'] = "qlmanage -t -f ".floor($p['thumb_size']/128)." -o /tmp/ ".$shellpath."; mv ".$tmp_path." ".$tpfile; - $cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size']; - - if (in_array($ext, $p['t_files']['sips'])) { - $external_tool = "sips"; - } elseif (in_array($ext, $p['t_files']['ffmpeg'])) { - $external_tool = "ffmpeg"; - } elseif (in_array($ext, $p['t_files']['sox'])) { - $external_tool = "sox"; - } else { - $external_tool = null; - } - - // using ql-tool or qlmanage may generate blank media icons - switch ($p['thumb_priority']) { - case 0: - // external tool priority - $priority = array($external_tool,"ql-thumbnail"); - break; - case 1: - // ql-thumbnail priority - $priority = array("ql-thumbnail",$external_tool,"qltool","qlmanage"); - break; - case 2: - // qltool priority - $priority = array("qltool","qlmanage","ql-thumbnail",$external_tool); - break; - case 3: - // qlmanage priority - $priority = array("qlmanage","qltool","ql-thumbnail",$external_tool); - break; - } - - $stmt->BindValue(":fid",$fid); - $stmt->BindValue(":created",time()); - // store the attempt in the DB so we don't try to generate thumbs for the same fid multiple times - - foreach ($priority as $tool) { - if (@$cmd[$tool]) { - $stmt->BindValue(":tool",$tool); - shell_exec($cmd[$tool]." 2>&1"); - msg($cmd[$tool]); - if (is_file($fmt[$tool]) && @filesize($fmt[$tool])) { - $stmt->BindValue(":relative_path",substr($fmt[$tool], strlen($bpath))); - list($width, $height) = getimagesize($fmt[$tool]); - $stmt->BindValue(":width",$width); - $stmt->BindValue(":height",$height); - break; - } - } - } - - $stmt->execute(); - + $tcmd = $bin_php." ".$helper." thumbs ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next(); + msg($tcmd); + $line[] = $tcmd; } - + + file_put_contents($batchfile,implode("\n", $line)); echo ProgressBar::finish($wopt_clear); - + + echo ProgressBar::start($passed_file,"Running thumb batch (".stepString().")"); + if ($wopt_parallel) { + passthru($bin_parallel." < ".$batchfile); + } else { + passthru("bash ".$batchfile); + } + echo ProgressBar::finish($wopt_clear); + } // Icons ////////////////////////////////////////// -$p['icons'] = 1; - if ($p['icons']) { echo ProgressBar::start($passed_file,"Creating icon batch (".stepString().")",1); - if (!is_dir("/tmp/Yuba/")) { mkdir("/tmp/Yuba/"); } - if (!is_dir("/tmp/Yuba/".$mytime)) { mkdir("/tmp/Yuba/".$mytime); } - $batchfile = "/tmp/Yuba/".$mytime."/_batch_icons.sh"; + $batchfile = $tmpdir."/_batch_icons.sh"; $helper = realpath("helper.php"); foreach ($fx as $count => $array) { @@ -711,14 +614,16 @@ if ($p['icons']) { msg($icmd); $line[] = $icmd; } + file_put_contents($batchfile,implode("\n", $line)); - echo ProgressBar::finish($wopt_clear); - echo ProgressBar::start($passed_file,"Running icon batch (".stepString().")"); - - passthru($bin_parallel." < ".$batchfile); - + echo ProgressBar::start($passed_file,"Running icon batch (".stepString().")"); + if ($wopt_parallel) { + passthru($bin_parallel." < ".$batchfile); + } else { + passthru("bash ".$batchfile); + } echo ProgressBar::finish($wopt_clear); } diff --git a/version.txt b/version.txt index fabfd2d..ef2481a 100755 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.7.12.6 \ No newline at end of file +0.7.12.7 \ No newline at end of file