diff --git a/CHANGELOG.md b/CHANGELOG.md index 68bf85e..61a7215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog All notable changes to this project will be documented in this file. Older changes are summarized on individual commits. +## [0.8.0.0] +- Rewrite icon generation +- Open in browser postflight + ## [0.7.13.2] - Added PHP extension checks - Moved icon, thumb, meta, contents into helper diff --git a/README.md b/README.md index 114c162..8d651ba 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ **Yuba** generates web-browsable catalogues from locally attached HFS+ filesystems. Its client application gathers forensic-quality data about a volume, properly interpreting bundles, reading Spotlight data, Finder flags, labels, and other contextual information. It can generate hashes, thumbnails, and gather 3rd party metadata with exiftool and mediainfo. Yuba's SQLite catalogues are comprehensive, lightweight, optimized for massive (1 million+) trees, and reflect incremental changes to contents and metadata. A server-side PHP script is provided, which allows familiar, Finder-style browsing of a catalogue. -####[⇩ Download Yuba 0.7.13.2](https://www.profiteroles.org/git/p/Yuba/raw/master/Yuba.app.zip) ([Changelog](CHANGELOG.md)) +####[⇩ Download Yuba 0.8.0.0](https://www.profiteroles.org/git/p/Yuba/raw/master/Yuba.app.zip) ([Changelog](CHANGELOG.md)) ## Features diff --git a/Tester.zip b/Tester.zip index 9c03815..f426738 100644 Binary files a/Tester.zip and b/Tester.zip differ diff --git a/Tester/Tester.php b/Tester/Tester.php index 0399bda..af6abb0 100644 --- a/Tester/Tester.php +++ b/Tester/Tester.php @@ -62,9 +62,17 @@ if ($mode == "thumb") { $cmd['ql-thumbnail'][] = $bin_qlthumb." ".$shellpath." ".$tprefix."qlthumb.jpg public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8"; $cmd['ql-thumbnail'][] = $tprefix."qlthumb.jpg"; - $cmd['sox'][] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 +dither -colors 16 ".$tprefix."sox.png"; - //$cmd['sox'][] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tprefix."sox.png"; - $cmd['sox'][] = $tprefix."sox.png"; + $cmd['sox_q'][] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_pngquant." - -o ".$tprefix."sox-quant.png"; + $cmd['sox_q'][] = $tprefix."sox-quant.png"; + + $cmd['sox_c'][] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -quality 85% ".$tprefix."sox-convert.jpg"; + $cmd['sox_c'][] = $tprefix."sox-convert.jpg"; + + $cmd['sox_cx'][] = $bin_sox." ".$shellpath." -n remix - trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -r -o - | ".$bin_convert." - -quality 85% ".$tprefix."sox-convert_x.jpg"; + $cmd['sox_cx'][] = $tprefix."sox-convert_x.jpg"; + + $cmd['sox_s'][] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".escapeshellarg($wprefix.basename($argv[2]).".png")."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($wprefix.basename($argv[2]).".png")." --out ".$tprefix."sox-sips.jpg"; + $cmd['sox_s'][] = $tprefix."sox-sips.jpg"; $cmd['ffmpeg'][] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='".$p['thumb_size'].":-2' -q:v 3 ".$tprefix."ffmpeg.jpg"; // only works properly on horizontal videos $cmd['ffmpeg'][] = $tprefix."ffmpeg.jpg"; @@ -74,10 +82,13 @@ if ($mode == "thumb") { $cmd['qltool'][] = $bin_qltool." di ".$shellpath." ".$p['icon_size']." ".$p['icon_size']." | base64 --decode | ".$bin_convert." - -scale 50% -strip -define png:compression-level=9 ".$tprefix."qltool.png"; $cmd['qltool'][] = $tprefix."qltool.png"; + $cmd['qltool_q'][] = $bin_qltool." di ".$shellpath." ".$p['icon_size']." ".$p['icon_size']." | base64 --decode | ".$bin_convert." - -scale 50% - | ".$bin_pngquant." - -o ".$tprefix."qltool-quant.png"; + $cmd['qltool_q'][] = $tprefix."qltool-quant.png"; + $cmd['ql-icon'][] = $bin_qlicon." --input=".$shellpath." --width=".$p['icon_size']." --height=".$p['icon_size']." --output=".$tprefix."qlicon.png"; $cmd['ql-icon'][] = $tprefix."qlicon.png"; - $cmd['ql-icon_q'][] = $bin_qlicon." --input=".$shellpath." --width=".$p['icon_size']." --height=".$p['icon_size']." | ".$bin_pngquant." - > ".$tprefix."qlicon-quant.png"; + $cmd['ql-icon_q'][] = $bin_qlicon." --input=".$shellpath." --width=".$p['icon_size']." --height=".$p['icon_size']." | ".$bin_pngquant." - -o ".$tprefix."qlicon-quant.png"; $cmd['ql-icon_q'][] = $tprefix."qlicon-quant.png"; $cmd['qlmanage_sx'][] = $bin_qlmanage." -ti -s ".$p['icon_size']." -o ".$wprefix." ".$shellpath." ; mv ".escapeshellarg($wprefix.basename($argv[2]).".png")." ".$tprefix."qlmanage-size.png"; @@ -105,6 +116,7 @@ foreach ($cmd as $exec) { if(!filesize($exec[1])) { echo "NO RESULT\n"; + unlink($exec[1]); } else { rename($exec[1],$dir."/".$i."_".$file."_".$elapsed."msec.".$ext); } @@ -113,5 +125,6 @@ foreach ($cmd as $exec) { } shell_exec("qlmanage -p ".$tprefix."/* > /dev/null 2>&1"); +shell_exec("open ".$tprefix); ?> \ No newline at end of file diff --git a/Yuba.app.zip b/Yuba.app.zip index 9292b5e..b96981b 100644 Binary files a/Yuba.app.zip and b/Yuba.app.zip differ diff --git a/Yuba.php b/Yuba.php index 12e4a1b..5e3eb4f 100755 --- a/Yuba.php +++ b/Yuba.php @@ -38,6 +38,7 @@ require("filetypes.php"); $wopt_steps = 8; // total number of steps $wopt_currstep = 1; if ($p['debug']) { $wopt_clear = 0; } else { $wopt_clear = 1; } +$wopt_debug_unique = 0; $parser = new plistParser(); @@ -98,10 +99,11 @@ if ($p['contents'] && !extension_loaded("zip")) { if (!isset($argv[1]) || $argv[1] == "") { echo "No input"; die; } $zpath = realpath(@$argv[1]); if ($p['bdest']) { $bdest = realpath($p['bdest']); } else { $bdest = "/Users/".get_current_user()."/Documents/Yuba/"; } -if (!is_dir($bdest)) { if (!mkdir($bdest)) { echo "Error creating directory: ".$bdest; die; } } +if (!is_dir($bdest)) { if (!mkdir($bdest,0777,true)) { echo "Error creating destination directory"; die; } } // Check for bundle if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); } +if ($wopt_debug_unique) { $blabel .= "-".$mytime; } if (is_writable($zpath)) { echo "Warning: source is writeable\n"; } $bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle"; @@ -660,12 +662,12 @@ foreach (array_merge($mb['t'],$mb['a']) as $item) { $ibuild[] = ":".$item; } +$dbo->exec("CREATE TABLE mdls (".implode(",",$cbuild).")"); + if ($p['spotlight']) { echo ProgressBar::start($passed_total,"Spotlight (".stepString().")"); - $dbo->exec("CREATE TABLE mdls (".implode(",",$cbuild).")"); - foreach ($files as $splFileInfo) { $path = $splFileInfo->getPathname(); @@ -887,7 +889,7 @@ foreach ($files as $splFileInfo) { } elseif ($type == "dir") { - if (@is_array($dpreview) && $dpreview[$pathname]) { + if (is_array($dpreview) && @$dpreview[$pathname]) { $fetch_thumb = $dbp->query("SELECT * FROM thumbs WHERE fid='".$dpreview[$pathname]."'")->fetch(); if (@$fetch_thumb['relative_path']) { $stmt->BindValue(":thumb_filename",$fetch_thumb['relative_path']); @@ -1036,9 +1038,11 @@ while ($row_a = $loop->fetch()) { } $m['m']['SkimTrackCount'] = @count($m_base); - foreach (@$m_base as $track) { - if (!@$m['m']['SkimDims'] && @$track['Width'] && @$track['Height']) { - $m['m']['SkimDims'] = @sanitize($track['Width'],"i").$display_delimiter.@sanitize($track['Height'],"i"); + if (is_array($m_base)) { + foreach (@$m_base as $track) { + if (!@$m['m']['SkimDims'] && @$track['Width'] && @$track['Height']) { + $m['m']['SkimDims'] = @sanitize($track['Width'],"i").$display_delimiter.@sanitize($track['Height'],"i"); + } } } @@ -1108,7 +1112,10 @@ $dbo->exec("UPDATE _skim SET status='completed_in_".$seconds."'"); // rsync -if ($p['postflight'] == 2 && $p['rsync_dest']) { +if ($p['postflight'] == 3) { + $url = "http://localhost/rtc.php?db=data/".basename($bpath)."/".$stamp.".sqlite3"; + exec("open ".$url); + } elseif ($p['postflight'] == 2 && $p['rsync_dest']) { echo msg("rsync..."); $command = "rsync -avv -e ssh ".$bpath." ".$p['rsync_dest']; $count = trim(shell_exec("find ".escapeshellarg($bpath)." | wc -l")); diff --git a/YubaPrefs.php b/YubaPrefs.php index 0bdf318..168c243 100644 --- a/YubaPrefs.php +++ b/YubaPrefs.php @@ -13,14 +13,6 @@ function makeWindowString($p, $strings) { *.title = Preferences *.floating = 1 - parallel.type = checkbox - parallel.label = Execute shell commands in parallel - parallel.default = ".$p['parallel']." - - debug.type = checkbox - debug.label = Enable verbose logging - debug.default = ".$p['debug']." - bdest.type = openbrowser bdest.filetype = directory bdest.label = Destination @@ -32,6 +24,7 @@ function makeWindowString($p, $strings) { postflight.option = ".$strings[0][0]." postflight.option = ".$strings[0][1]." postflight.option = ".$strings[0][2]." + postflight.option = ".$strings[0][3]." postflight.default = ".$strings[0][$p['postflight']]." postflight.width = 380 @@ -41,6 +34,21 @@ function makeWindowString($p, $strings) { rsync_dest.placeholder = user@server.com:files/ rsync_dest.width = 380 + hr0.type = image + hr0.path = ".__DIR__."/hr.png"." + hr0.width = 380 + hr0.height = 2 + + parallel.type = checkbox + parallel.label = Use parallel + parallel.default = ".$p['parallel']." + + debug.type = checkbox + debug.label = Verbose logging + debug.default = ".$p['debug']." + debug.x = 200 + debug.y = 439 + hr1.type = image hr1.path = ".__DIR__."/hr.png"." hr1.width = 380 @@ -79,15 +87,15 @@ function makeWindowString($p, $strings) { thumbs.default = ".$strings[2][$p['thumbs']]." thumbs.width = 120 - thumb_priority.type = popup - thumb_priority.label = Priority - thumb_priority.option = ".$strings[3][0]." - thumb_priority.option = ".$strings[3][1]." - thumb_priority.option = ".$strings[3][2]." - thumb_priority.default = ".$strings[3][$p['thumb_priority']]." - thumb_priority.width = 120 - thumb_priority.x = 150 - thumb_priority.y = 227 + thumb_mode.type = popup + thumb_mode.label = Mode + thumb_mode.option = ".$strings[3][0]." + thumb_mode.option = ".$strings[3][1]." + thumb_mode.option = ".$strings[3][2]." + thumb_mode.default = ".$strings[3][$p['thumb_mode']]." + thumb_mode.width = 120 + thumb_mode.x = 150 + thumb_mode.y = 227 thumb_size.type = textfield thumb_size.default = ".$p['thumb_size']." @@ -105,14 +113,15 @@ function makeWindowString($p, $strings) { icons.default = ".$strings[2][$p['icons']]." icons.width = 120 - icon_tool.type = popup - icon_tool.label = Tool - icon_tool.option = ".$strings[4][0]." - icon_tool.option = ".$strings[4][1]." - icon_tool.default = ".$strings[4][$p['icon_tool']]." - icon_tool.width = 120 - icon_tool.x = 150 - icon_tool.y = 162 + icon_mode.type = popup + icon_mode.label = Mode + icon_mode.option = ".$strings[4][0]." + icon_mode.option = ".$strings[4][1]." + icon_mode.option = ".$strings[4][2]." + icon_mode.default = ".$strings[4][$p['icon_mode']]." + icon_mode.width = 120 + icon_mode.x = 150 + icon_mode.y = 162 icon_size.type = textfield icon_size.default = ".$p['icon_size']." @@ -198,11 +207,11 @@ if(!$p['bdest']) { // Load strings -$strings[] = array("Do nothing","Reveal result in Finder","Upload result with rsync"); +$strings[] = array("Do nothing","Reveal result in Finder","Upload result with rsync","Open via localhost URL"); $strings[] = array("Skip","Collect","Collect & verify","Collect & repair atimes"); $strings[] = array("Skip","Generate","Rebuild"); -$strings[] = array("external","ql-thumbnail","qlmanage"); -$strings[] = array("qltool","qlmanage"); +$strings[] = array("All files","Most files","Some files"); +$strings[] = array("All files","Most files","Some files"); // Launch Pashua and parse results @@ -229,9 +238,9 @@ if (@$result['cb']) { $result['postflight'] = array_search($result['postflight'],$strings[0]); $result['stat_mode'] = array_search($result['stat_mode'],$strings[1]); $result['thumbs'] = array_search($result['thumbs'],$strings[2]); -$result['thumb_priority'] = array_search($result['thumb_priority'],$strings[3]); +$result['thumb_mode'] = array_search($result['thumb_mode'],$strings[3]); $result['icons'] = array_search($result['icons'],$strings[2]); -$result['icon_tool'] = array_search($result['icon_tool'],$strings[4]); +$result['icon_mode'] = array_search($result['icon_mode'],$strings[4]); // If the user didn't specify a destpath, set to default diff --git a/current_version.txt b/current_version.txt index 9b7b06b..01eb481 100755 --- a/current_version.txt +++ b/current_version.txt @@ -1 +1 @@ -0.7.13.2 \ No newline at end of file +0.8.0.0 \ No newline at end of file diff --git a/filetypes.php b/filetypes.php index edee792..8ccb82f 100755 --- a/filetypes.php +++ b/filetypes.php @@ -9,11 +9,13 @@ $bin_mediainfo = escapeshellarg(__DIR__."/bin/mediainfo"); $bin_exiftool = escapeshellarg(__DIR__."/bin/exiftool"); $bin_ffmpeg = escapeshellarg(__DIR__."/bin/ffmpeg"); $bin_qlthumb = escapeshellarg(__DIR__."/bin/ql-thumbnail"); +$bin_qlicon = escapeshellarg(__DIR__."/bin/ql-icon"); $bin_qltool = escapeshellarg(__DIR__."/bin/qltool"); $bin_sox = escapeshellarg(__DIR__."/bin/sox"); -$bin_pngcrush = escapeshellarg(__DIR__."/bin/pngcrush"); +$bin_pngquant = escapeshellarg(__DIR__."/bin/pngquant"); $bin_parallel = escapeshellarg(__DIR__."/bin/parallel"); $bin_convert = escapeshellarg(__DIR__."/bin/convert"); +$bin_flacicon = escapeshellarg(__DIR__."/bin/flacicon"); // System tools $bin_php = "php"; @@ -73,10 +75,6 @@ $p['c_files'] = array( "DS_Store", "doc", "docx" ); -// thumbnail bindings -$p['t_f']['pdf'][] = "jpg"; -$p['t_f']['pdf'][] = array("gs","qlmanage","sips","vips"); - // Use ffmpeg to generate thumbnails for these files $p['t_files']['ffmpeg'] = array( "mkv", "avi", @@ -136,9 +134,6 @@ $p['i_skip'] = array( "jpg", "zip", "DS_Store" ); -// Custom qlgenerator bindings -$p['i_bindings']['indd'] = array("com.adobe.indesign-document","Art View.qlgenerator"); - // Run mediainfo on these files $p['m_files'] = array( "mkv", "ogg", diff --git a/functions.php b/functions.php index 2ffaa1f..bfe0744 100755 --- a/functions.php +++ b/functions.php @@ -102,11 +102,11 @@ function askMulti($string, $buttons) { return array_search($result,$buttons); } -function msg($string) { +function msg($string, $extra=true) { global $messages_log_file; $logstring = "[".date('Y-m-d h:i:s')."] ".$string."\n"; file_put_contents($messages_log_file, $logstring, FILE_APPEND); - file_put_contents("/tmp/yuba/debug.log", $logstring, FILE_APPEND); + if ($extra) { file_put_contents("/tmp/yuba/debug.log", $logstring, FILE_APPEND); } return $string."\n"; } diff --git a/helper.php b/helper.php index 7a51955..79c4b27 100755 --- a/helper.php +++ b/helper.php @@ -13,9 +13,14 @@ require("filetypes.php"); $dm = ""; function dfm ($string, $flag = 0) { // dumb workaround for debug messages appearing out of order + // bypass for single-thread to observe hangs + //dm($string); + //return global $dm; - if ($flag) { + if ($flag == 1) { $dm .= str_repeat("-",33)."\n".$string."\n".str_repeat("-",33)."\n"; + } elseif ($flag == 2) { + $dm .= str_repeat("-",66)."\n".$string."\n".str_repeat("-",66)."\n"; } else { $dm .= "[".date('Y-m-d h:i:s')."] ".$string."\n"; } @@ -25,6 +30,7 @@ $fid = $argv[1]; $pathname = $argv[2]; $shellpath = escapeshellarg($pathname); $ext = pathinfo($pathname,PATHINFO_EXTENSION); +$file = pathinfo($pathname, PATHINFO_BASENAME); $bpath = $argv[3]; $mytime = $argv[4]; @@ -34,9 +40,6 @@ $messages_log_file = $bpath."/".$stamp."_messages.log"; $tmpdir = "/tmp/yuba/".$mytime; if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); dfm("mkdir ",$tmpdir); } -$file = pathinfo($pathname, PATHINFO_BASENAME); -$ext = pathinfo($pathname, PATHINFO_EXTENSION); - $dbp = new PDO("sqlite:".$bpath."/pool.sqlite3"); $dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbp->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); @@ -49,9 +52,9 @@ $dbp->query("PRAGMA journal_mode = WAL"); $sbatch = array(); -$estring = "\nBATCH: ".basename($pathname); +$estring = "\nBATCH: ".shortlabel(basename($pathname),20); -dfm("BATCH INIT ".$file.":".print_r($argv,true)); +dfm("BATCH INIT ".$file.":".print_r($argv,true),2); // Thumbs ////////////////////////////////////////// @@ -62,7 +65,7 @@ if ($p['thumbs']) { $estring .= " THUMBS"; - // "bad" filesizes + // "bad" filesizes from generic icons $discard = array( 3953, 4977, 5019, @@ -70,13 +73,13 @@ if ($p['thumbs']) { 6616, 17393 ); - $tfile = $tmpdir."/".substr($fid,0,2)."/".$fid.".jpg"; - $tpfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention + $tfile = $tmpdir."/thumbs/".substr($fid,0,2)."/".$fid.".jpg"; + $tpfile = $tmpdir."/thumbs/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention $dfile = $bpath."/thumbs/".substr($fid, 0, 2)."/".$fid.".jpg";; - if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); dfm("mkdir ".dirname($tfile)); } + if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile),0777,true); dfm("mkdir ".dirname($tfile)); } - if (in_array($ext, $p['t_skip'])) { + if ($p['thumb_mode'] != 0 && in_array($ext, $p['t_skip'])) { // file extension is in the skip list $estring .= " ->t_skip"; dfm("file extension is in the skip list"); @@ -89,10 +92,11 @@ if ($p['thumbs']) { $estring .= " ->skip_fast"; goto icons; } - - $rowid = @$dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()['rowid']; // check for existing thumb generation attempt + $sql = "SELECT rowid FROM thumbs WHERE fid='".$fid."'"; + dfm($sql); + $rowid = @$dbp->query($sql)->fetch()['rowid']; if (!$rowid) { // no prior attempt @@ -113,7 +117,7 @@ if ($p['thumbs']) { // prior attempt dfm("a prior attempt at thumb generation was found for ".$fid); - $estring .= " ->skip"; + $estring .= " ->pskip"; goto icons; } @@ -121,8 +125,7 @@ if ($p['thumbs']) { $cmd['sips'] = $bin_sips." -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; // add ?? "-d profile --deleteColorManagementProperties" $cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size']; - //$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tfile; - $cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile; + $cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/4}') spectrogram -r -o - | ".$bin_convert." - -quality 85% ".$tfile; $cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile; $cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8"; @@ -138,43 +141,47 @@ if ($p['thumbs']) { $external_tool = null; } dfm("external tool for ".$ext." = ".$external_tool); - switch ($p['thumb_priority']) { - // external tool priority + switch ($p['thumb_mode']) { case 0: $priority = array($external_tool,"ql-thumbnail","qlmanage"); break; - // ql-thumbnail priority - case 1: $priority = array("ql-thumbnail","qlmanage",$external_tool); break; - // qlmanage priority - case 2: $priority = array("qlmanage","ql-thumbnail",$external_tool); break; + case 1: $priority = array("ql-thumbnail","qlmanage"); break; + case 2: $priority = array("ql-thumbnail"); break; } + dfm("thumb mode ".$p['thumb_mode']); dfm("priority for ".$ext." is: ".print_r($priority,true)); $stmt->BindValue(":fid",$fid); $stmt->BindValue(":created",time()); - if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); dfm("mkdir ".dirname($dfile)); } - foreach ($priority as $tool) { - if (empty($cmd[$tool])) { break; } + if (empty($cmd[$tool])) { continue; } - shell_exec($cmd[$tool]." 2>&1"); dfm($cmd[$tool]); + shell_exec($cmd[$tool]." 2>&1"); $estring .= " ->".$tool; - + $checksize = @filesize($tfile); - if ($checksize && !in_array($checksize,$discard)) { - $estring .= " ->use"; - $stmt->BindValue(":tool",$tool); - $stmt->BindValue(":relative_path",substr($dfile, strlen($bpath))); - list($width, $height) = getimagesize($tfile); - $stmt->BindValue(":width",$width); - $stmt->BindValue(":height",$height); - rename($tfile,$dfile); - dfm("moving ".$tfile." to ".$dfile); - break; + if ($checksize) { + if (!empty($discard) && in_array($checksize,$discard)) { + $estring .= " ->discard"; + dfm($checksize." is in the banned size array"); + } else { + $estring .= " ->use"; + dfm ("generated ".$tfile." filesize is ".$checksize); + $stmt->BindValue(":tool",$tool); + $stmt->BindValue(":relative_path",substr($dfile, strlen($bpath))); + dfm("getimagesize on ".$tfile); + list($width, $height) = getimagesize($tfile); + $stmt->BindValue(":width",$width); + $stmt->BindValue(":height",$height); + if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); dfm("mkdir ".dirname($dfile)); } + rename($tfile,$dfile); + dfm("moving ".$tfile." to ".$dfile); + break; + } } else { $estring .= " ->discard"; - dfm("filesize ".$checksize." is in the banned size array (or is zero)"); + dfm($tool." produced a bad file (size=".$checksize.")"); } } @@ -193,109 +200,129 @@ if ($p['icons']) { dfm("ICONS",1); $estring .= " ICONS"; - - if ($p['icon_tool'] == 0) { - $tool = "qltool"; - } elseif ($p['icon_tool'] == 1) { - $tool = "qlmanage"; - } - - if (in_array($ext, $p['i_skip'])) { - // file extension is in the skip list + + if ($p['icon_mode'] != 0 && in_array($ext, $p['i_skip'])) { $estring .= " ->i_skip"; + dfm("file extension is in the skip list"); goto meta; } - $tfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; - if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); } - // workaround for qlmanage naming convention + $tfile = $tmpdir."/icons/".substr($fid,0,2)."/".$fid.".png"; + if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile),0777,true); dfm("mkdir ".dirname($tfile)); } - $rowid = @$dbp->query("SELECT rowid FROM icons WHERE fid='".$fid."'")->fetch()['rowid']; - // check for existing icon generation attempt + // we can't check for files (faster) here because they are stored by md5_file hash + $sql = "SELECT rowid FROM icons WHERE fid='".$fid."'"; + dfm($sql); + $rowid = @$dbp->query($sql)->fetch()['rowid']; if (!$rowid) { - // no prior attempt + dfm("no prior attempt was found"); $stmt = $dbp->prepare("INSERT INTO icons VALUES (:fid, :hash, :created, :relative_path, :tool)"); - $estring .= " ->generate"; } elseif ($rowid && $p['icons'] == 2) { - // prior attempt but rebuild mode + dfm("a prior attempt was found but we are in rebuild mode"); $stmt = $dbp->prepare("UPDATE icons SET fid = :fid, hash = :hash, created = :created, relative_path = :relative_path, tool = :tool WHERE rowid = ".$rowid); $estring .= " ->rebuild"; } else { - // prior attempt - $estring .= " ->skip"; + dfm("a prior attempt was found, skipping"); + $estring .= " ->pskip"; goto meta; } $stmt->BindValue(":fid",$fid); - - if ($tool == "qltool") { - - $cmd = $bin_qltool." di ".$shellpath." ".$p['icon_size']." ".$p['icon_size']." | base64 --decode > ".escapeshellarg($tfile); - - } elseif ($tool == "qlmanage") { - - // > code for custom qlgenerator bindings goes here < - - $cmd = $bin_qlmanage." -ti -s ".$p['icon_size']." -o ".dirname($tfile)." ".$shellpath; - //$cmd = $bin_qlmanage." -ti -f ".floor($p['icon_size']/128)." -o ".dirname($tfile)." ".$shellpath; - - } - shell_exec($cmd); - - if (@filesize($tfile)) { - - $hash = md5_file($tfile); - $stmt->BindValue(":hash",$hash); - - if ($row = @$dbp->query("SELECT * FROM icons WHERE hash='".$hash."'")->fetchAll()[0]) { - // check for another icon with the same hash - - $stmt->BindValue(":created",$row['created']); - $stmt->BindValue(":relative_path",$row['relative_path']); - $stmt->BindValue(":tool",$row['tool']); - - $estring .= " ->recycle"; - - } else { - - $dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png"; + $cmd['flacicon'] = $bin_flacicon." ".$shellpath." ".$p['icon_size']." ".$tfile; + $cmd['ql-icon'] = $bin_qlicon." --input=".$shellpath." --width=".$p['icon_size']." --height=".$p['icon_size']." | ".$bin_pngquant." - -o ".$tfile; + $cmd['qltool'] = $bin_qltool." di ".$shellpath." ".$p['icon_size']." ".$p['icon_size']." | base64 --decode | ".$bin_convert." - -scale 50% - | ".$bin_pngquant." - -o ".$tfile; + + if ($p['icon_mode'] == 2) { + + dfm("icon mode 2, bypassing hash check"); + $tool = "ql-icon"; + shell_exec($cmd[$tool]." 2>&1"); + $stmt->BindValue(":tool",$tool); + dfm($cmd[$tool]); + $estring .= " ->".$tool; + + $checksize = @filesize($tfile); + if ($checksize) { + dfm("generated ".$tfile." filesize is ".$checksize); + $dfile = $bpath."/icons/".substr($fid, 0, 2)."/".$fid.".png"; if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); } - + rename($tfile,$dfile); + dfm("moving ".$tfile." to ".$dfile); + $stmt->BindValue(":created",time()); + $stmt->BindValue(":relative_path",substr($dfile,strlen($bpath))); + $stmt->BindValue(":hash",null); $estring .= " ->use"; + } else { + $estring .= " ->discard"; + dfm($tool." produced a bad file (size=".$checksize.")"); + } + + } else { + + if ($ext == "flac") { + $tools = array("flacicon","ql-icon","qltool"); + } else { + $tools = array("ql-icon","qltool"); + } + foreach ($tools as $tool) { + if (!@filesize($tfile)) { + shell_exec($cmd[$tool]." 2>&1"); + dfm($cmd[$tool]); + $estring .= " ->".$tool; + } + } + + $checksize = @filesize($tfile); + if ($checksize) { + + dfm ("generated ".$tfile." filesize is ".$checksize); + dfm("md5_file on ".$tfile); + $hash = md5_file($tfile); + $stmt->BindValue(":hash",$hash); + + $sql = "SELECT * FROM icons WHERE hash='".$hash."'"; + dfm($sql); + $row = @$dbp->query($sql)->fetchAll()[0]; - if ($tool == "qltool") { - // qltool makes 2x icons for some reason - - //shell_exec($bin_sips." -z ".$p['icon_size']." ".$p['icon_size']." ".$tfile." --out ".$dfile); - shell_exec($bin_convert." ".escapeshellarg($tfile)." -scale 50% -strip -define png:compression-level=9 ".$dfile); + if ($row) { + + dfm("found another record with the same hash"); + $estring .= " ->recycle"; + $stmt->BindValue(":created",$row['created']); + $stmt->BindValue(":relative_path",$row['relative_path']); + $stmt->BindValue(":tool",$row['tool']); } else { - //rename($tfile,$dfile); - shell_exec($bin_convert." ".escapeshellarg($tfile)." -strip -define png:compression-level=9 ".$dfile); - + dfm("no existing hash record"); + $dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png"; + if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); } + rename($tfile,$dfile); + dfm("moving ".$tfile." to ".$dfile); + $stmt->BindValue(":created",time()); + $stmt->BindValue(":relative_path",substr($dfile,strlen($bpath))); + $stmt->BindValue(":tool",$tool); + $estring .= " ->use"; + } - - $stmt->BindValue(":created",time()); - $stmt->BindValue(":relative_path",substr($dfile,strlen($bpath))); - $stmt->BindValue(":tool",$tool); - - } - - } else { - $estring .= " ->discard"; - - } + } else { + $estring .= " ->discard"; + dfm($tool." produced a bad file (size=".$checksize.")"); + + } + + } + $sbatch[] = $stmt; $estring .= " ->db"; @@ -313,15 +340,19 @@ if ($p['meta']) { if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) { $estring .= " ->notmedia"; + dfm("file extension ".$ext." is not a media file"); goto hashy; } if (in_array($ext, $p['e_files'])) { - $check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch(); - if (!reset($check)) { // hacky but needed because we set FETCH_ASSOC - $arrstring = shell_exec($bin_exiftool." -php ".$shellpath); - // $rawexif = eval("return ".`$bin_exiftool -php $shellpath`); - // do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'" + $sql = "SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')"; + dfm($sql); + $check = @$dbp->query($sql)->fetch(); + if (!reset($check)) { + dfm("creating exiftool dump"); + $cmd = $bin_exiftool." -php ".$shellpath; + dfm($cmd); + $arrstring = shell_exec($cmd); if (substr($arrstring,0,5) == "Array") { $rawexif = eval("return ".$arrstring); $stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)"); @@ -332,21 +363,24 @@ if ($p['meta']) { } } else { $estring .= " ->e_found"; + dfm("existing exiftool dump found"); } } if (in_array($ext, $p['m_files'])) { - $check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch(); + $sql = "SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')"; + dfm($sql); + $check = @$dbp->query($sql)->fetch(); if (!reset($check)) { + dfm("creating mediainfo dump"); $stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)"); $stmt->BindValue(":fid",$fid); - //$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")))); - //$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")); $stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1")); $sbatch[] = $stmt; $estring .= " ->minfo"; } else { $estring .= " ->m_found"; + dfm("existing mediainfo dump found"); } } @@ -362,14 +396,20 @@ if ($p['hash']) { $estring .= " HASH"; $size = filesize($pathname); + dfm("file ".$pathname." is ".$size); $limit = $p['hash_limit']*1000000000; - $check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch(); + $sql = "SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')"; + dfm($sql); + $check = @$dbp->query($sql)->fetch(); if (reset($check)) { $estring .= " ->exists"; + dfm("hash already exists for ".$pathname); } elseif ($p['hash_limit'] && ($size > $limit)) { $estring .= " ->too big"; + dfm("file is too big to hash"); } else { $estring .= " ->generating"; + dfm("generating hash for ".$pathname); $stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)"); $stmt->BindValue(":fid",$fid); $stmt->BindValue(":hash",md5_file($pathname)); @@ -392,17 +432,21 @@ if ($p['contents']) { $cpath = $bpath."/contents/".substr($fid, 0, 2); $cfile = $cpath."/".$fid.".zip"; if (in_array($ext, $p['c_files'])) { - if (!is_dir($cpath)) { mkdir($cpath); } $max_size = $p['contents_limit'] * 1000; if ($ext == ".DS_Store") { // store all DS_Store files no matter how big + dfm("flagging ds_store file for inclusion"); $my_size = 1; } else { $my_size = filesize($pathname); - } + dfm("file ".$pathname." is ".$my_size); + } if (!file_exists($cfile) && $my_size < $max_size) { - + + if (!is_dir($cpath)) { mkdir($cpath); dfm("mkdir ",$cpath); } + $estring .= " ->zip"; + dfm("creating a zip archive of ".$pathname); $zip = new ZipArchive(); $zip->open($cfile, ZipArchive::CREATE); @@ -421,9 +465,11 @@ if ($p['contents']) { // Inserts ////////////////////////////////////////// -foreach ($sbatch as $stmt) { $stmt->execute(); } +dfm("SQL",1); + +foreach ($sbatch as $stmt) { $stmt->execute(); dfm($stmt->queryString); } dm($dm); -echo msg($estring); +echo msg($estring,false); ?> \ No newline at end of file diff --git a/prefs.php b/prefs.php index fba2d97..aaa81c4 100644 --- a/prefs.php +++ b/prefs.php @@ -1 +1 @@ -a:20:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"stat_mode";i:1;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:14:"thumb_priority";i:0;s:5:"icons";i:1;s:9:"icon_size";i:512;s:9:"icon_tool";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;s:8:"parallel";i:1;s:5:"debug";i:0;} \ No newline at end of file +a:20:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"stat_mode";i:1;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:10:"thumb_mode";i:0;s:5:"icons";i:1;s:9:"icon_size";i:256;s:9:"icon_mode";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;s:8:"parallel";i:1;s:5:"debug";i:0;} \ No newline at end of file diff --git a/web/rtc.php b/web/rtc.php index 07119c9..8fd9ea3 100644 --- a/web/rtc.php +++ b/web/rtc.php @@ -4,7 +4,7 @@ // Yuba RTC Browser ///////////////////////////////////////////////////////////////// -$browser_version = "0.7.13.0"; +$browser_version = "0.7.13.2"; require "togggle.php"; require "lib/ref/ref.php"; @@ -12,7 +12,7 @@ require "lib/ref/ref.php"; ref::config('expLvl', 1); ref::config('validHtml', TRUE); -$db_dir = "data/debug"; +$db_dir = "data/"; $icon_size = 96; $pad = 40; $border_tools = array("sips","sox","ffmpeg","ql-thumbnail"); @@ -61,7 +61,7 @@ div.iibox { display: table-cell; div.diibox { display: table-cell; vertical-align: middle; - background: url("/icons/directory.png"); + background: url("/yicons/directory.png"); background-repeat: no-repeat; background-size: px; background-position: center; @@ -105,8 +105,7 @@ img#thumb { padding: 6px; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.1); } -img#item { float: left; margin-right: 10px; width: 32px; height: 32px; } -img { margin-bottom: 8px; } +img#item { float: left; margin-right: 10px; width: 32px; height: 32px; margin-bottom: 8px; } div.title { padding: 0px 10px 0px 10px; } @@ -348,8 +347,8 @@ function mb_shortlabel($filename, $max = 40) { function findicon($filename) { $ext = pathinfo($filename)['extension']; - $good = "icons/null.png"; - foreach (glob("icons/*.png") as $file) { + $good = "yicons/null.png"; + foreach (glob("yicons/*.png") as $file) { if (pathinfo($file)['filename'] == $ext) { $good = $file; } @@ -359,9 +358,9 @@ function findicon($filename) { ///////////////////////////////////////////////////////////////// -$db_file = $_GET['db']; -$pid = $_GET['pid']; -$search = $_POST['query']; +$db_file = @$_GET['db']; +$pid = @$_GET['pid']; +$search = @$_POST['query']; $parser = new plistParser(); if ($db_file) { @@ -380,6 +379,8 @@ if ($db_file) { $zpath = $dbo->query("SELECT zpath FROM _skim")->fetch()['zpath']; $skim_version = $dbo->query("SELECT version FROM _skim")->fetch()['version']; + $spotlight_status = $dbo->query("SELECT mdutil FROM _skim")->fetch()['mdutil']; + // Check for initial view $view = $dbo->query("SELECT * FROM _skim")->fetch(); @@ -394,7 +395,7 @@ if ($db_file) { $view['qlmanage'] = array($view['qlmanage']); } $view['profile'] = array($view['profile']); - if ($view['details']) { + if (@$view['details']) { $view['details'] = unserialize($view['details']); } if (substr($view['disks'],0,5) == "parseString(utf8_for_xml($view['vdisks'])); } - if ($view['hdiutil']) { + if (@$view['hdiutil']) { $view['hdiutil'] = $parser->parseString(utf8_for_xml($view['hdiutil'])); } $view['Type'] = "dir"; @@ -419,12 +420,14 @@ if ($db_file) { if (is_serial($view['stat'])) { $view['stat'] = unserialize($view['stat']); } - $dirmdls = $dbo->query("SELECT rowid, * FROM mdls WHERE (rowid='".$view['rowid']."')")->fetch(); + if (!strpos($spotlight_status,"disabled")) { + $dirmdls = $dbo->query("SELECT rowid, * FROM mdls WHERE (rowid='".$view['rowid']."')")->fetch(); + } } // Header - echo "Home ► "; + echo "Home ► "; $dbs = glob(pathinfo($_GET['db'],PATHINFO_DIRNAME)."/????-??-??_??-??-??.sqlite3"); rsort($dbs); echo "