diff --git a/CHANGELOG.md b/CHANGELOG.md index ba95590..68bf85e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,18 @@ # Changelog All notable changes to this project will be documented in this file. Older changes are summarized on individual commits. +## [0.7.13.2] +- Added PHP extension checks +- Moved icon, thumb, meta, contents into helper +- Fixed numerous bugs in helper +- Parallel option in prefs +- Debug messaging and console +- Set WAL on dbp for performance + ## [0.7.13.1] - Thumb creation bugfix - App filepath bugfixes +- Switch to shell wrapper (to enable php bundling) ## [0.7.13.0] - Rewrites to diskutil code diff --git a/Console/Console.platypus b/Console/Console.platypus new file mode 100644 index 0000000..609456b --- /dev/null +++ b/Console/Console.platypus @@ -0,0 +1,65 @@ + + + + + AcceptsFiles + + AcceptsText + + Authentication + + Author + profiteroles + BundledFiles + + Creator + Platypus-5.2 + DeclareService + + DevelopmentVersion + + Droppable + + IconPath + /Volumes/Code/Yuba/master/Console/icon.icns + Identifier + org.profiteroles.Console + InterfaceType + Text Window + InterpreterPath + /bin/bash + Name + Console + NibPath + /usr/local/share/platypus/MainMenu.nib + OptimizeApplication + + Overwrite + + PromptForFileOnLaunch + + RemainRunning + + RunInBackground + + ScriptPath + /Volumes/Code/Yuba/master/Console/tail.sh + TextBackground + #23030e + TextFont + SFMono-Regular + TextForeground + #ffc3bd + TextSize + 11 + UniformTypes + + public.item + public.folder + + UseXMLPlistFormat + + Version + 0.1.0.0 + + diff --git a/Console/icon.icns b/Console/icon.icns new file mode 100644 index 0000000..2e4f312 Binary files /dev/null and b/Console/icon.icns differ diff --git a/Console/tail.sh b/Console/tail.sh new file mode 100755 index 0000000..3f86bc0 --- /dev/null +++ b/Console/tail.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +tail -F $1 \ No newline at end of file diff --git a/MainMenu.nib/designable.nib b/MainMenu.nib/designable.nib index 1f8472c..af81718 100644 --- a/MainMenu.nib/designable.nib +++ b/MainMenu.nib/designable.nib @@ -163,8 +163,7 @@ Gw - - + @@ -488,10 +487,15 @@ Gw - + + + + + + - + @@ -544,7 +548,7 @@ Gw - + diff --git a/MainMenu.nib/keyedobjects.nib b/MainMenu.nib/keyedobjects.nib index 65b70bb..5697209 100644 Binary files a/MainMenu.nib/keyedobjects.nib and b/MainMenu.nib/keyedobjects.nib differ diff --git a/Tester.zip b/Tester.zip index 97a0d98..ffa5731 100644 Binary files a/Tester.zip and b/Tester.zip differ diff --git a/Yuba.app.zip b/Yuba.app.zip index 4539d88..f2a49e8 100644 Binary files a/Yuba.app.zip and b/Yuba.app.zip differ diff --git a/Yuba.php b/Yuba.php index 5b0cd9b..8bce29c 100755 --- a/Yuba.php +++ b/Yuba.php @@ -7,6 +7,7 @@ $version = file_get_contents(__DIR__."/current_version.txt"); ini_set('memory_limit', '10240M'); date_default_timezone_set("America/Los_Angeles"); +if (!file_exists("/tmp/yuba/debug.log")) { touch("/tmp/yuba/debug.log"); } // Includes & Prefs ////////////////////////////////////////// @@ -25,10 +26,9 @@ require("filetypes.php"); // Manual prefs -$wopt_steps = 12; // total number of steps +$wopt_steps = 8; // total number of steps $wopt_currstep = 1; -$wopt_clear = 0; // set to 0 for debug -$wopt_parallel = 1; // use parallel? +if ($p['debug']) { $wopt_clear = 0; } else { $wopt_clear = 1; } $parser = new plistParser(); @@ -42,6 +42,13 @@ if (@$argv[1] == "Preferences...") { die; } +// Console + +if (@$argv[1] == "Console") { + exec("open -n ".__DIR__."/bin/Console.app --args /tmp/yuba/debug.log"); + die; + } + // Version check if (@$argv[1] == "Check for Updates...") { @@ -59,13 +66,29 @@ if (@$argv[1] == "Check for Updates...") { } +dm("Launching Yuba\n".str_repeat("-",33)."\n".print_r($p,true)); + +// PHP Checks +////////////////////////////////////////// + +$needed = array("iconv","fileinfo","json","PDO","pdo_sqlite","SimpleXML","sqlite3","xml"); +foreach ($needed as $ext) { + if (!extension_loaded($ext)) { + alert("PHP is missing the ".$ext.". Exiting.","PHP Extension Missing"); + echo "QUITAPP\n"; + } + } +if ($p['contents'] && !extension_loaded("zip")) { + alert("PHP is missing the zip extension. Yuba will not collect file contents.","PHP Extension Missing"); + revise_prefs(array("contents" => 0)); + } + // Path & application variables ////////////////////////////////////////// $mytime = time(); $tmpdir = "/tmp/yuba/".$mytime; if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); } - $stamp = date("Y-m-d_H-i-s", $mytime); if (!isset($argv[1]) || $argv[1] == "") { echo "No input"; die; } @@ -199,10 +222,10 @@ $dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3"); $dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); -$dbo->query("PRAGMA page_size = 4096"); -$dbo->query("PRAGMA cache_size = 10000"); -$dbo->query("PRAGMA locking_mode = EXCLUSIVE"); -$dbo->query("PRAGMA synchronous = NORMAL"); +//$dbo->query("PRAGMA page_size = 4096"); +//$dbo->query("PRAGMA cache_size = 10000"); +//$dbo->query("PRAGMA locking_mode = EXCLUSIVE"); +//$dbo->query("PRAGMA synchronous = NORMAL"); $dbo->query("PRAGMA journal_mode = WAL"); $dbo->exec("CREATE TABLE _skim ( @@ -376,12 +399,6 @@ $dbp = new PDO("sqlite:".$bpath."/pool.sqlite3"); $dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); -//$dbp->query("PRAGMA page_size = 4096"); -//$dbp->query("PRAGMA cache_size = 10000"); -//$dbp->query("PRAGMA locking_mode = EXCLUSIVE"); -//$dbp->query("PRAGMA synchronous = NORMAL"); -//$dbp->query("PRAGMA journal_mode = WAL"); - $dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)"); $dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)"); $dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)"); @@ -554,29 +571,29 @@ foreach ($fx as $array) { } echo ProgressBar::finish($wopt_clear); -// Thumbs +// Helper ////////////////////////////////////////// -if ($p['thumbs']) { +if ($p['thumbs'] || $p['icons'] || $p['meta'] || $p['hash'] || $p['contents']) { - echo ProgressBar::start($passed_file,"Creating thumb batch (".stepString().")",1); + echo ProgressBar::start($passed_file,"Creating batch (".stepString().")",1); - $batchfile = $tmpdir."/_batch_thumbs.sh"; + $batchfile = $tmpdir."/_batch.sh"; $helper = realpath("helper.php"); foreach ($fx as $count => $array) { $fid = $array[0]; $pathname = $array[1]; - $tcmd = $bin_php." ".escapeshellarg($helper)." thumbs ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next(); + $tcmd = $bin_php." ".escapeshellarg($helper)." ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; printf '\\n".ProgressBar::next()."'"; msg($tcmd); - $tline[] = $tcmd; + $line[] = $tcmd; } - file_put_contents($batchfile,implode("\n", $tline)); + file_put_contents($batchfile,implode("\n", $line)); echo ProgressBar::finish($wopt_clear); - echo ProgressBar::start($passed_file,"Running thumb batch (".stepString().")"); - if ($wopt_parallel) { + echo ProgressBar::start($passed_file,"Running batch (".stepString().")"); + if ($p['parallel']) { passthru($bin_parallel." < ".$batchfile); } else { passthru("bash ".$batchfile); @@ -585,183 +602,6 @@ if ($p['thumbs']) { } -// Icons -////////////////////////////////////////// - -if ($p['icons']) { - - echo ProgressBar::start($passed_file,"Creating icon batch (".stepString().")",1); - - $batchfile = $tmpdir."/_batch_icons.sh"; - $helper = realpath("helper.php"); - - foreach ($fx as $count => $array) { - $fid = $array[0]; - $pathname = $array[1]; - $icmd = $bin_php." ".escapeshellarg($helper)." icons ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next(); - msg($icmd); - $iline[] = $icmd; - } - - file_put_contents($batchfile,implode("\n", $iline)); - echo ProgressBar::finish($wopt_clear); - - echo ProgressBar::start($passed_file,"Running icon batch (".stepString().")"); - if ($wopt_parallel) { - passthru($bin_parallel." < ".$batchfile); - } else { - passthru("bash ".$batchfile); - } - echo ProgressBar::finish($wopt_clear); - - } - -// External metadata -////////////////////////////////////////// - -if ($p['meta']) { - - echo ProgressBar::start(count($fx),"Collecting external metadata (".stepString().")"); - - foreach ($fx as $array) { - - $fid = $array[0]; - $pathname = $array[1]; - $shellpath = escapeshellarg($pathname); - $ext = pathinfo($pathname,PATHINFO_EXTENSION); - $found = 1; - - if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) { - echo ProgressBar::next("Not a media file: ".shortlabel($pathname)); - continue; - } - - if (in_array($ext, $p['e_files'])) { - $check = $dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch()[0]; - if (!$check) { - $arrstring = shell_exec($bin_exiftool." -php ".$shellpath); - // $rawexif = eval("return ".`$bin_exiftool -php $shellpath`); - // do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'" - if (substr($arrstring,0,5) == "Array") { - $rawexif = eval("return ".$arrstring); - $stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)"); - $stmt->BindValue(":fid",$fid); - $stmt->BindValue(":tags",serialize($rawexif[0])); - $stmt->execute(); - $found = 0; - } - } - } - - if (in_array($ext, $p['m_files'])) { - $check = $dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch()[0]; - if (!$check) { - $stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)"); - $stmt->BindValue(":fid",$fid); - //$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")))); - //$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")); - $stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1")); - $stmt->execute(); - $found = 0; - } - } - - if ($found) { - echo ProgressBar::next("Metadata found: ".shortlabel($pathname)); - } else { - echo ProgressBar::next("Collecting metadata: ".shortlabel($pathname)); - } - - } - - echo ProgressBar::finish($wopt_clear); - - } - -// Hashes -////////////////////////////////////////// - -if ($p['hash']) { - - if ($p['hash_limit']) { - $message = "Generating hashes for files under ".$p['hash_limit']."GB"; - } else { - $message = "Generating hashes for all files"; - } - - echo ProgressBar::start(count($fx),$message." (".stepString().")"); - - foreach ($fx as $array) { - $fid = $array[0]; - $pathname = $array[1]; - $size = filesize($pathname); - $limit = $p['hash_limit']*1000000000; - $check = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0]; - if ($check) { - echo ProgressBar::next("Hash already exists: ".shortlabel($pathname)); - } elseif ($p['hash_limit'] && ($size > $limit)) { - echo ProgressBar::next("Too big to hash: ".shortlabel($pathname)." (".human_filesize($size).")"); - } else { - echo ProgressBar::next("Generating hash: ".shortlabel($pathname)); - $stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)"); - $stmt->BindValue(":fid",$fid); - $stmt->BindValue(":hash",md5_file($pathname)); - $stmt->execute(); - } - } - - echo ProgressBar::finish($wopt_clear); - - } - -// Contents -////////////////////////////////////////// - -if ($p['contents']) { - - // should be rewritten to check against filemtimes - - echo ProgressBar::start(count($fx),"Gathering contents (".stepString().")"); - - foreach ($fx as $array) { - $fid = $array[0]; - $pathname = $array[1]; - $ext = pathinfo($pathname,PATHINFO_EXTENSION); - $cpath = $bpath."/contents/".substr($fid, 0, 2); - $cfile = $cpath."/".$fid.".zip"; - if (in_array($ext, $p['c_files'])) { - if (!is_dir($cpath)) { mkdir($cpath); } - $max_size = $p['contents_limit'] * 1000; - if ($ext == ".DS_Store") { - // store all DS_Store files no matter how big - $my_size = 1; - } else { - $my_size = filesize($pathname); - } - if (!file_exists($cfile) && $my_size < $max_size) { - - msg("Zipping ".$pathname); - - $zip = new ZipArchive(); - $zip->open($cfile, ZipArchive::CREATE); - $zip->addfile($pathname,basename($pathname)); - $zip->close(); - - $stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)"); - $stmt->BindValue(":fid",$fid); - $stmt->BindValue(":created",time()); - $stmt->BindValue(":relative_path",substr($cfile, strlen($bpath))); - $stmt->execute(); - - } - } - echo ProgressBar::next(true); - } - - echo ProgressBar::finish($wopt_clear); - - } - // Pool Indices ////////////////////////////////////////// diff --git a/YubaPrefs.php b/YubaPrefs.php index 6366fcc..0bdf318 100644 --- a/YubaPrefs.php +++ b/YubaPrefs.php @@ -13,6 +13,14 @@ function makeWindowString($p, $strings) { *.title = Preferences *.floating = 1 + parallel.type = checkbox + parallel.label = Execute shell commands in parallel + parallel.default = ".$p['parallel']." + + debug.type = checkbox + debug.label = Enable verbose logging + debug.default = ".$p['debug']." + bdest.type = openbrowser bdest.filetype = directory bdest.label = Destination diff --git a/current_version.txt b/current_version.txt index 7f92c14..9b7b06b 100755 --- a/current_version.txt +++ b/current_version.txt @@ -1 +1 @@ -0.7.13.1 \ No newline at end of file +0.7.13.2 \ No newline at end of file diff --git a/functions.php b/functions.php index ef43b7e..9f516b4 100755 --- a/functions.php +++ b/functions.php @@ -76,6 +76,13 @@ class ProgressBar { // Functions ////////////////////////////////////////// +function reviseprefs($array) { // bad practice + $prefs_file = "/Users/".get_current_user()."/Library/Preferences/org.profiteroles.Yuba.php"; + $p = unserialize(file_get_contents($prefs_file)); + foreach ($array as $key => $value) { $p[$key] = "value"; } + file_put_contents($prefs_file,serialize($result)); + } + function prettysize($size) { $unit=array('b','kb','mb','gb','tb','pb'); return @round($size/pow(1024,($i=floor(log($size,1024)))),2).' '.$unit[$i]; @@ -99,9 +106,20 @@ function msg($string) { global $messages_log_file; $logstring = "[".date('Y-m-d h:i:s')."] ".$string."\n"; file_put_contents($messages_log_file, $logstring, FILE_APPEND); + file_put_contents("/tmp/yuba/debug.log", $logstring, FILE_APPEND); return $string."\n"; } +function dm($string) { + global $p; if (!$p['debug']) { return; } // bad practice + if (strpos($string,"\n") === false) { + $logstring = "[".date('Y-m-d h:i:s')."] ".$string."\n"; + } else { + $logstring = "[".date('Y-m-d h:i:s')."]\n".$string."\n".str_repeat("-",33)."\n"; + } + file_put_contents("/tmp/yuba/debug.log", $logstring, FILE_APPEND); + } + function timeToSeconds($val) { if (!is_numeric($val) && strpos($val,":") === false) { $val = str_replace(" s","",$val); diff --git a/helper.php b/helper.php index ec97cc0..7a51955 100755 --- a/helper.php +++ b/helper.php @@ -10,31 +10,189 @@ $p = unserialize(file_get_contents($prefs_file)); require("functions.php"); require("filetypes.php"); -$mode = $argv[1]; -$fid = $argv[2]; -$pathname = $argv[3]; -$bpath = $argv[4]; -$mytime = $argv[5]; +$dm = ""; +function dfm ($string, $flag = 0) { + // dumb workaround for debug messages appearing out of order + global $dm; + if ($flag) { + $dm .= str_repeat("-",33)."\n".$string."\n".str_repeat("-",33)."\n"; + } else { + $dm .= "[".date('Y-m-d h:i:s')."] ".$string."\n"; + } + } + +$fid = $argv[1]; +$pathname = $argv[2]; +$shellpath = escapeshellarg($pathname); +$ext = pathinfo($pathname,PATHINFO_EXTENSION); +$bpath = $argv[3]; +$mytime = $argv[4]; + +$stamp = date("Y-m-d_H-i-s", $mytime); +$messages_log_file = $bpath."/".$stamp."_messages.log"; $tmpdir = "/tmp/yuba/".$mytime; -if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); } +if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); dfm("mkdir ",$tmpdir); } $file = pathinfo($pathname, PATHINFO_BASENAME); $ext = pathinfo($pathname, PATHINFO_EXTENSION); -$shellpath = escapeshellarg($pathname); - $dbp = new PDO("sqlite:".$bpath."/pool.sqlite3"); $dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbp->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); -echo "\nBatch (".$mode."): ".basename($pathname); +//$dbp->query("PRAGMA page_size = 4096"); +//$dbp->query("PRAGMA cache_size = 10000"); +//$dbp->query("PRAGMA locking_mode = EXCLUSIVE"); +//$dbp->query("PRAGMA synchronous = NORMAL"); +$dbp->query("PRAGMA journal_mode = WAL"); -switch ($mode) { +$sbatch = array(); + +$estring = "\nBATCH: ".basename($pathname); + +dfm("BATCH INIT ".$file.":".print_r($argv,true)); + +// Thumbs ////////////////////////////////////////// + +thumbs: +if ($p['thumbs']) { - // Icons ////////////////////////////////////////// + dfm("THUMBS",1); + + $estring .= " THUMBS"; + + // "bad" filesizes + $discard = array( 3953, + 4977, + 5019, + 6059, + 6616, + 17393 ); + + $tfile = $tmpdir."/".substr($fid,0,2)."/".$fid.".jpg"; + $tpfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention + $dfile = $bpath."/thumbs/".substr($fid, 0, 2)."/".$fid.".jpg";; + + if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); dfm("mkdir ".dirname($tfile)); } + + if (in_array($ext, $p['t_skip'])) { + // file extension is in the skip list + $estring .= " ->t_skip"; + dfm("file extension is in the skip list"); + goto icons; + } + + if ($p['thumbs'] == 1 && file_exists($dfile)) { + // if this is not a rebuild, first check for an existing thumb file (faster) + dfm("this is not a rebuild, and an existing thumb file was found"); + $estring .= " ->skip_fast"; + goto icons; + } + + $rowid = @$dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()['rowid']; + + // check for existing thumb generation attempt + + if (!$rowid) { + // no prior attempt + dfm("no prior attempt at thumb file generation was made for ".$fid); + + $stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)"); + dfm("mysql insert for thumb generation attempt"); + $estring .= " ->generate"; + + } elseif ($rowid && $p['thumbs'] == 2) { + // prior attempt but rebuild mode + + $stmt = $dbp->prepare("UPDATE thumbs SET fid = :fid, created = :created, relative_path = :relative_path, width = :width, height = :height, tool = :tool WHERE rowid = ".$rowid); + dfm("mysql update for rebuild"); + $estring .= " ->rebuild"; + + } else { + // prior attempt + dfm("a prior attempt at thumb generation was found for ".$fid); + + $estring .= " ->skip"; + goto icons; + + } + + $cmd['sips'] = $bin_sips." -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; // add ?? "-d profile --deleteColorManagementProperties" + $cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size']; + + //$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tfile; + $cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile; + $cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile; + + $cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8"; + $cmd['qlmanage'] = $bin_qlmanage." -t -s ".$p['thumb_size']." -o ".dirname($tpfile)." ".$shellpath."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile; + + if (in_array($ext, $p['t_files']['sips'])) { + $external_tool = "sips"; + } elseif (in_array($ext, $p['t_files']['ffmpeg'])) { + $external_tool = "ffmpeg"; + } elseif (in_array($ext, $p['t_files']['sox'])) { + $external_tool = "sox"; + } else { + $external_tool = null; + } + dfm("external tool for ".$ext." = ".$external_tool); + switch ($p['thumb_priority']) { + // external tool priority + case 0: $priority = array($external_tool,"ql-thumbnail","qlmanage"); break; + // ql-thumbnail priority + case 1: $priority = array("ql-thumbnail","qlmanage",$external_tool); break; + // qlmanage priority + case 2: $priority = array("qlmanage","ql-thumbnail",$external_tool); break; + } + dfm("priority for ".$ext." is: ".print_r($priority,true)); + + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":created",time()); + + if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); dfm("mkdir ".dirname($dfile)); } + + foreach ($priority as $tool) { - case "icons": + if (empty($cmd[$tool])) { break; } + + shell_exec($cmd[$tool]." 2>&1"); + dfm($cmd[$tool]); + $estring .= " ->".$tool; + + $checksize = @filesize($tfile); + if ($checksize && !in_array($checksize,$discard)) { + $estring .= " ->use"; + $stmt->BindValue(":tool",$tool); + $stmt->BindValue(":relative_path",substr($dfile, strlen($bpath))); + list($width, $height) = getimagesize($tfile); + $stmt->BindValue(":width",$width); + $stmt->BindValue(":height",$height); + rename($tfile,$dfile); + dfm("moving ".$tfile." to ".$dfile); + break; + } else { + $estring .= " ->discard"; + dfm("filesize ".$checksize." is in the banned size array (or is zero)"); + } + + } + + $sbatch[] = $stmt; + + $estring .= " ->db"; + + } + +// Icons ////////////////////////////////////////// + +icons: +if ($p['icons']) { + + dfm("ICONS",1); + + $estring .= " ICONS"; if ($p['icon_tool'] == 0) { $tool = "qltool"; @@ -44,8 +202,8 @@ switch ($mode) { if (in_array($ext, $p['i_skip'])) { // file extension is in the skip list - echo " ->i_skip"; - break; + $estring .= " ->i_skip"; + goto meta; } $tfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; @@ -59,19 +217,19 @@ switch ($mode) { // no prior attempt $stmt = $dbp->prepare("INSERT INTO icons VALUES (:fid, :hash, :created, :relative_path, :tool)"); - echo " ->generate"; + $estring .= " ->generate"; } elseif ($rowid && $p['icons'] == 2) { // prior attempt but rebuild mode $stmt = $dbp->prepare("UPDATE icons SET fid = :fid, hash = :hash, created = :created, relative_path = :relative_path, tool = :tool WHERE rowid = ".$rowid); - echo " ->rebuild"; + $estring .= " ->rebuild"; } else { // prior attempt - echo " ->skip"; - break; + $estring .= " ->skip"; + goto meta; } @@ -104,14 +262,14 @@ switch ($mode) { $stmt->BindValue(":relative_path",$row['relative_path']); $stmt->BindValue(":tool",$row['tool']); - echo " ->recycle"; + $estring .= " ->recycle"; } else { $dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png"; if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); } - echo " ->use"; + $estring .= " ->use"; if ($tool == "qltool") { // qltool makes 2x icons for some reason @@ -134,132 +292,138 @@ switch ($mode) { } else { - echo " ->discard"; + $estring .= " ->discard"; } - $stmt->execute(); + $sbatch[] = $stmt; - echo " ->db"; - - break; - - case "thumbs": - - // Thumbs ////////////////////////////////////////// - - // "bad" filesizes - $discard = array( 3953, - 4977, - 5019, - 6059, - 6616, - 17393 ); - - $tfile = $tmpdir."/".substr($fid,0,2)."/".$fid.".jpg"; - $tpfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention - if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); } - - if (in_array($ext, $p['t_skip'])) { - // file extension is in the skip list - echo " ->t_skip"; - break; - } - - if ($p['thumbs'] == 1 && file_exists($tfile)) { - // if this is not a rebuild, first check for an existing thumb file (faster) - echo " ->skip_fast"; - break; - } - - $rowid = @$dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()['rowid']; - // check for existing thumb generation attempt - - if (!$rowid) { - // no prior attempt - - $stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)"); - echo " ->generate"; - - } elseif ($rowid && $p['thumbs'] == 2) { - // prior attempt but rebuild mode - - $stmt = $dbp->prepare("UPDATE thumbs SET fid = :fid, created = :created, relative_path = :relative_path, width = :width, height = :height, tool = :tool WHERE rowid = ".$rowid); - echo " ->rebuild"; - - } else { - // prior attempt - - echo " ->skip"; - break; - - } - - $cmd['sips'] = $bin_sips." -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; // add ?? "-d profile --deleteColorManagementProperties" - $cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size']; - - //$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tfile; - $cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile; - $cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile; - - $cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8"; - $cmd['qlmanage'] = $bin_qlmanage." -t -s ".$p['thumb_size']." -o ".dirname($tpfile)." ".$shellpath."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile; - - if (in_array($ext, $p['t_files']['sips'])) { - $external_tool = "sips"; - } elseif (in_array($ext, $p['t_files']['ffmpeg'])) { - $external_tool = "ffmpeg"; - } elseif (in_array($ext, $p['t_files']['sox'])) { - $external_tool = "sox"; - } else { - $external_tool = null; - } - switch ($p['thumb_priority']) { - // external tool priority - case 0: $priority = array($external_tool,"ql-thumbnail","qlmanage"); break; - // ql-thumbnail priority - case 1: $priority = array("ql-thumbnail","qlmanage",$external_tool); break; - // qlmanage priority - case 2: $priority = array("qlmanage","ql-thumbnail",$external_tool); break; - } - - $stmt->BindValue(":fid",$fid); - $stmt->BindValue(":created",time()); - - $dfile = $bpath."/thumbs/".substr($fid, 0, 2)."/".$fid.".jpg";; - if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); } - - foreach ($priority as $tool) { - - if (empty($cmd[$tool])) { continue; } - - shell_exec($cmd[$tool]." 2>&1"); - echo " ->".$tool; - - $checksize = @filesize($tfile); - if ($checksize && !in_array($checksize,$discard)) { - echo " ->use"; - $stmt->BindValue(":tool",$tool); - $stmt->BindValue(":relative_path",substr($dfile, strlen($bpath))); - list($width, $height) = getimagesize($tfile); - $stmt->BindValue(":width",$width); - $stmt->BindValue(":height",$height); - rename($tfile,$dfile); - break; - } else { - echo " ->discard"; - } - - } - - $stmt->execute(); - - echo " ->db"; - - break; + $estring .= " ->db"; } -echo "\n"; +// Metadata ////////////////////////////////////////// + +meta: +if ($p['meta']) { + + dfm("META",1); + + $estring .= " META"; + + if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) { + $estring .= " ->notmedia"; + goto hashy; + } + + if (in_array($ext, $p['e_files'])) { + $check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch(); + if (!reset($check)) { // hacky but needed because we set FETCH_ASSOC + $arrstring = shell_exec($bin_exiftool." -php ".$shellpath); + // $rawexif = eval("return ".`$bin_exiftool -php $shellpath`); + // do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'" + if (substr($arrstring,0,5) == "Array") { + $rawexif = eval("return ".$arrstring); + $stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":tags",serialize($rawexif[0])); + $sbatch[] = $stmt; + $estring .= " ->etool"; + } + } else { + $estring .= " ->e_found"; + } + } + + if (in_array($ext, $p['m_files'])) { + $check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch(); + if (!reset($check)) { + $stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)"); + $stmt->BindValue(":fid",$fid); + //$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")))); + //$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")); + $stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1")); + $sbatch[] = $stmt; + $estring .= " ->minfo"; + } else { + $estring .= " ->m_found"; + } + } + + } + +// Hash ////////////////////////////////////////// + +hashy: +if ($p['hash']) { + + dfm("HASH",1); + + $estring .= " HASH"; + + $size = filesize($pathname); + $limit = $p['hash_limit']*1000000000; + $check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch(); + if (reset($check)) { + $estring .= " ->exists"; + } elseif ($p['hash_limit'] && ($size > $limit)) { + $estring .= " ->too big"; + } else { + $estring .= " ->generating"; + $stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":hash",md5_file($pathname)); + $sbatch[] = $stmt; + } + + $estring .= " ->done"; + + } + +// Contents ////////////////////////////////////////// + +contents: +if ($p['contents']) { + + dfm("CONTENTS",1); + + $estring .= " CONTENTS"; + + $cpath = $bpath."/contents/".substr($fid, 0, 2); + $cfile = $cpath."/".$fid.".zip"; + if (in_array($ext, $p['c_files'])) { + if (!is_dir($cpath)) { mkdir($cpath); } + $max_size = $p['contents_limit'] * 1000; + if ($ext == ".DS_Store") { + // store all DS_Store files no matter how big + $my_size = 1; + } else { + $my_size = filesize($pathname); + } + if (!file_exists($cfile) && $my_size < $max_size) { + + $estring .= " ->zip"; + + $zip = new ZipArchive(); + $zip->open($cfile, ZipArchive::CREATE); + $zip->addfile($pathname,basename($pathname)); + $zip->close(); + + $stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":created",time()); + $stmt->BindValue(":relative_path",substr($cfile, strlen($bpath))); + $sbatch[] = $stmt; + + } + } + } + +// Inserts ////////////////////////////////////////// + +foreach ($sbatch as $stmt) { $stmt->execute(); } + +dm($dm); +echo msg($estring); ?> \ No newline at end of file diff --git a/prefs.php b/prefs.php index e44516e..fba2d97 100644 --- a/prefs.php +++ b/prefs.php @@ -1 +1 @@ -a:18:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"stat_mode";i:1;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:14:"thumb_priority";i:0;s:5:"icons";i:1;s:9:"icon_size";i:512;s:9:"icon_tool";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;} \ No newline at end of file +a:20:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"stat_mode";i:1;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:14:"thumb_priority";i:0;s:5:"icons";i:1;s:9:"icon_size";i:512;s:9:"icon_tool";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;s:8:"parallel";i:1;s:5:"debug";i:0;} \ No newline at end of file