p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.7.13.2

Browse Source
This commit is contained in:
profiteroles
2019-10-09 04:07:46 -07:00
parent 197c100c01
commit e325062915
14 changed files with 458 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@@ -1,9 +1,18 @@
# Changelog # Changelog
All notable changes to this project will be documented in this file. Older changes are summarized on individual commits. All notable changes to this project will be documented in this file. Older changes are summarized on individual commits.
## [0.7.13.2]
- Added PHP extension checks
- Moved icon, thumb, meta, contents into helper
- Fixed numerous bugs in helper
- Parallel option in prefs
- Debug messaging and console
- Set WAL on dbp for performance
## [0.7.13.1] ## [0.7.13.1]
- Thumb creation bugfix - Thumb creation bugfix
- App filepath bugfixes - App filepath bugfixes
- Switch to shell wrapper (to enable php bundling)
## [0.7.13.0] ## [0.7.13.0]
- Rewrites to diskutil code - Rewrites to diskutil code

65
Console/Console.platypus Normal file
View File

@@ -0,0 +1,65 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>AcceptsFiles</key>
<false/>
<key>AcceptsText</key>
<false/>
<key>Authentication</key>
<false/>
<key>Author</key>
<string>profiteroles</string>
<key>BundledFiles</key>
<array/>
<key>Creator</key>
<string>Platypus-5.2</string>
<key>DeclareService</key>
<false/>
<key>DevelopmentVersion</key>
<false/>
<key>Droppable</key>
<false/>
<key>IconPath</key>
<string>/Volumes/Code/Yuba/master/Console/icon.icns</string>
<key>Identifier</key>
<string>org.profiteroles.Console</string>
<key>InterfaceType</key>
<string>Text Window</string>
<key>InterpreterPath</key>
<string>/bin/bash</string>
<key>Name</key>
<string>Console</string>
<key>NibPath</key>
<string>/usr/local/share/platypus/MainMenu.nib</string>
<key>OptimizeApplication</key>
<true/>
<key>Overwrite</key>
<false/>
<key>PromptForFileOnLaunch</key>
<false/>
<key>RemainRunning</key>
<false/>
<key>RunInBackground</key>
<true/>
<key>ScriptPath</key>
<string>/Volumes/Code/Yuba/master/Console/tail.sh</string>
<key>TextBackground</key>
<string>#23030e</string>
<key>TextFont</key>
<string>SFMono-Regular</string>
<key>TextForeground</key>
<string>#ffc3bd</string>
<key>TextSize</key>
<real>11</real>
<key>UniformTypes</key>
<array>
<string>public.item</string>
<string>public.folder</string>
</array>
<key>UseXMLPlistFormat</key>
<false/>
<key>Version</key>
<string>0.1.0.0</string>
</dict>
</plist>

BIN
Console/icon.icns Normal file
View File

Binary file not shown.

3
Console/tail.sh Executable file
View File

@@ -0,0 +1,3 @@
#!/bin/bash
tail -F $1

14
MainMenu.nib/designable.nib generated
View File

@@ -163,8 +163,7 @@ Gw
<action selector="menuItemSelected:" target="207" id="x6l-Tq-pHp"/> <action selector="menuItemSelected:" target="207" id="x6l-Tq-pHp"/>
</connections> </connections>
</menuItem> </menuItem>
<menuItem title="Check for Updates..." id="jWa-5b-mBK" userLabel="Check for updates..."> <menuItem title="Check for Updates..." keyEquivalent="u" id="jWa-5b-mBK" userLabel="Check for updates...">
<modifierMask key="keyEquivalentModifierMask"/>
<connections> <connections>
<action selector="menuItemSelected:" target="207" id="VBb-a8-Cku"/> <action selector="menuItemSelected:" target="207" id="VBb-a8-Cku"/>
</connections> </connections>
@@ -488,10 +487,15 @@ Gw
<menuItem title="Window" id="19"> <menuItem title="Window" id="19">
<menu key="submenu" title="Window" systemMenu="window" id="24"> <menu key="submenu" title="Window" systemMenu="window" id="24">
<items> <items>
<menuItem title="Zoom" keyEquivalent="z" id="197"> <menuItem title="Console" keyEquivalent="0" id="ZMj-KE-Xmg">
<connections>
<action selector="menuItemSelected:" target="207" id="uTf-9L-ThU"/>
</connections>
</menuItem>
<menuItem title="Zoom" keyEquivalent="z" id="bf1-wK-fgP">
<modifierMask key="keyEquivalentModifierMask" shift="YES" command="YES"/> <modifierMask key="keyEquivalentModifierMask" shift="YES" command="YES"/>
<connections> <connections>
<action selector="performZoom:" target="-1" id="198"/> <action selector="performZoom:" target="-1" id="xLL-JF-YCG"/>
</connections> </connections>
</menuItem> </menuItem>
<menuItem title="Minimize" keyEquivalent="m" id="23"> <menuItem title="Minimize" keyEquivalent="m" id="23">
@@ -544,7 +548,7 @@ Gw
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/> <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<clipView key="contentView" drawsBackground="NO" copiesOnScroll="NO" id="Hk1-72-Owa"> <clipView key="contentView" drawsBackground="NO" copiesOnScroll="NO" id="Hk1-72-Owa">
<rect key="frame" x="1" y="1" width="559" height="365"/> <rect key="frame" x="1" y="1" width="559" height="365"/>
<autoresizingMask key="autoresizingMask"/> <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<subviews> <subviews>
<textView editable="NO" importsGraphics="NO" richText="NO" verticallyResizable="YES" usesFontPanel="YES" findStyle="panel" allowsDocumentBackgroundColorChange="YES" linkDetection="YES" id="225"> <textView editable="NO" importsGraphics="NO" richText="NO" verticallyResizable="YES" usesFontPanel="YES" findStyle="panel" allowsDocumentBackgroundColorChange="YES" linkDetection="YES" id="225">
<rect key="frame" x="0.0" y="0.0" width="559" height="365"/> <rect key="frame" x="0.0" y="0.0" width="559" height="365"/>

BIN
MainMenu.nib/keyedobjects.nib generated
View File

Binary file not shown.

BIN
Tester.zip
View File

Binary file not shown.

BIN
Yuba.app.zip
View File

Binary file not shown.

240
Yuba.php
View File

@@ -7,6 +7,7 @@ $version = file_get_contents(__DIR__."/current_version.txt");
ini_set('memory_limit', '10240M'); ini_set('memory_limit', '10240M');
date_default_timezone_set("America/Los_Angeles"); date_default_timezone_set("America/Los_Angeles");
if (!file_exists("/tmp/yuba/debug.log")) { touch("/tmp/yuba/debug.log"); }
// Includes & Prefs // Includes & Prefs
////////////////////////////////////////// //////////////////////////////////////////
@@ -25,10 +26,9 @@ require("filetypes.php");
// Manual prefs // Manual prefs
$wopt_steps = 12; // total number of steps $wopt_steps = 8; // total number of steps
$wopt_currstep = 1; $wopt_currstep = 1;
$wopt_clear = 0; // set to 0 for debug if ($p['debug']) { $wopt_clear = 0; } else { $wopt_clear = 1; }
$wopt_parallel = 1; // use parallel?
$parser = new plistParser(); $parser = new plistParser();
@@ -42,6 +42,13 @@ if (@$argv[1] == "Preferences...") {
die; die;
} }
// Console
if (@$argv[1] == "Console") {
exec("open -n ".__DIR__."/bin/Console.app --args /tmp/yuba/debug.log");
die;
}
// Version check // Version check
if (@$argv[1] == "Check for Updates...") { if (@$argv[1] == "Check for Updates...") {
@@ -59,13 +66,29 @@ if (@$argv[1] == "Check for Updates...") {
} }
dm("Launching Yuba\n".str_repeat("-",33)."\n".print_r($p,true));
// PHP Checks
//////////////////////////////////////////
$needed = array("iconv","fileinfo","json","PDO","pdo_sqlite","SimpleXML","sqlite3","xml");
foreach ($needed as $ext) {
if (!extension_loaded($ext)) {
alert("PHP is missing the ".$ext.". Exiting.","PHP Extension Missing");
echo "QUITAPP\n";
}
}
if ($p['contents'] && !extension_loaded("zip")) {
alert("PHP is missing the zip extension. Yuba will not collect file contents.","PHP Extension Missing");
revise_prefs(array("contents" => 0));
}
// Path & application variables // Path & application variables
////////////////////////////////////////// //////////////////////////////////////////
$mytime = time(); $mytime = time();
$tmpdir = "/tmp/yuba/".$mytime; $tmpdir = "/tmp/yuba/".$mytime;
if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); } if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); }
$stamp = date("Y-m-d_H-i-s", $mytime); $stamp = date("Y-m-d_H-i-s", $mytime);
if (!isset($argv[1]) || $argv[1] == "") { echo "No input"; die; } if (!isset($argv[1]) || $argv[1] == "") { echo "No input"; die; }
@@ -199,10 +222,10 @@ $dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3");
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); $dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$dbo->query("PRAGMA page_size = 4096"); //$dbo->query("PRAGMA page_size = 4096");
$dbo->query("PRAGMA cache_size = 10000"); //$dbo->query("PRAGMA cache_size = 10000");
$dbo->query("PRAGMA locking_mode = EXCLUSIVE"); //$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
$dbo->query("PRAGMA synchronous = NORMAL"); //$dbo->query("PRAGMA synchronous = NORMAL");
$dbo->query("PRAGMA journal_mode = WAL"); $dbo->query("PRAGMA journal_mode = WAL");
$dbo->exec("CREATE TABLE _skim ( $dbo->exec("CREATE TABLE _skim (
@@ -376,12 +399,6 @@ $dbp = new PDO("sqlite:".$bpath."/pool.sqlite3");
$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
//$dbp->query("PRAGMA page_size = 4096");
//$dbp->query("PRAGMA cache_size = 10000");
//$dbp->query("PRAGMA locking_mode = EXCLUSIVE");
//$dbp->query("PRAGMA synchronous = NORMAL");
//$dbp->query("PRAGMA journal_mode = WAL");
$dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)"); $dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)"); $dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)"); $dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)");
@@ -554,29 +571,29 @@ foreach ($fx as $array) {
} }
echo ProgressBar::finish($wopt_clear); echo ProgressBar::finish($wopt_clear);
// Thumbs // Helper
////////////////////////////////////////// //////////////////////////////////////////
if ($p['thumbs']) { if ($p['thumbs'] || $p['icons'] || $p['meta'] || $p['hash'] || $p['contents']) {
echo ProgressBar::start($passed_file,"Creating thumb batch (".stepString().")",1); echo ProgressBar::start($passed_file,"Creating batch (".stepString().")",1);
$batchfile = $tmpdir."/_batch_thumbs.sh"; $batchfile = $tmpdir."/_batch.sh";
$helper = realpath("helper.php"); $helper = realpath("helper.php");
foreach ($fx as $count => $array) { foreach ($fx as $count => $array) {
$fid = $array[0]; $fid = $array[0];
$pathname = $array[1]; $pathname = $array[1];
$tcmd = $bin_php." ".escapeshellarg($helper)." thumbs ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next(); $tcmd = $bin_php." ".escapeshellarg($helper)." ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; printf '\\n".ProgressBar::next()."'";
msg($tcmd); msg($tcmd);
$tline[] = $tcmd; $line[] = $tcmd;
} }
file_put_contents($batchfile,implode("\n", $tline)); file_put_contents($batchfile,implode("\n", $line));
echo ProgressBar::finish($wopt_clear); echo ProgressBar::finish($wopt_clear);
echo ProgressBar::start($passed_file,"Running thumb batch (".stepString().")"); echo ProgressBar::start($passed_file,"Running batch (".stepString().")");
if ($wopt_parallel) { if ($p['parallel']) {
passthru($bin_parallel." < ".$batchfile); passthru($bin_parallel." < ".$batchfile);
} else { } else {
passthru("bash ".$batchfile); passthru("bash ".$batchfile);
@@ -585,183 +602,6 @@ if ($p['thumbs']) {
} }
// Icons
//////////////////////////////////////////
if ($p['icons']) {
echo ProgressBar::start($passed_file,"Creating icon batch (".stepString().")",1);
$batchfile = $tmpdir."/_batch_icons.sh";
$helper = realpath("helper.php");
foreach ($fx as $count => $array) {
$fid = $array[0];
$pathname = $array[1];
$icmd = $bin_php." ".escapeshellarg($helper)." icons ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next();
msg($icmd);
$iline[] = $icmd;
}
file_put_contents($batchfile,implode("\n", $iline));
echo ProgressBar::finish($wopt_clear);
echo ProgressBar::start($passed_file,"Running icon batch (".stepString().")");
if ($wopt_parallel) {
passthru($bin_parallel." < ".$batchfile);
} else {
passthru("bash ".$batchfile);
}
echo ProgressBar::finish($wopt_clear);
}
// External metadata
//////////////////////////////////////////
if ($p['meta']) {
echo ProgressBar::start(count($fx),"Collecting external metadata (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$shellpath = escapeshellarg($pathname);
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$found = 1;
if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) {
echo ProgressBar::next("Not a media file: ".shortlabel($pathname));
continue;
}
if (in_array($ext, $p['e_files'])) {
$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch()[0];
if (!$check) {
$arrstring = shell_exec($bin_exiftool." -php ".$shellpath);
// $rawexif = eval("return ".`$bin_exiftool -php $shellpath`);
// do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'"
if (substr($arrstring,0,5) == "Array") {
$rawexif = eval("return ".$arrstring);
$stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":tags",serialize($rawexif[0]));
$stmt->execute();
$found = 0;
}
}
}
if (in_array($ext, $p['m_files'])) {
$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch()[0];
if (!$check) {
$stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)");
$stmt->BindValue(":fid",$fid);
//$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"))));
//$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"));
$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1"));
$stmt->execute();
$found = 0;
}
}
if ($found) {
echo ProgressBar::next("Metadata found: ".shortlabel($pathname));
} else {
echo ProgressBar::next("Collecting metadata: ".shortlabel($pathname));
}
}
echo ProgressBar::finish($wopt_clear);
}
// Hashes
//////////////////////////////////////////
if ($p['hash']) {
if ($p['hash_limit']) {
$message = "Generating hashes for files under ".$p['hash_limit']."GB";
} else {
$message = "Generating hashes for all files";
}
echo ProgressBar::start(count($fx),$message." (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$size = filesize($pathname);
$limit = $p['hash_limit']*1000000000;
$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0];
if ($check) {
echo ProgressBar::next("Hash already exists: ".shortlabel($pathname));
} elseif ($p['hash_limit'] && ($size > $limit)) {
echo ProgressBar::next("Too big to hash: ".shortlabel($pathname)." (".human_filesize($size).")");
} else {
echo ProgressBar::next("Generating hash: ".shortlabel($pathname));
$stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":hash",md5_file($pathname));
$stmt->execute();
}
}
echo ProgressBar::finish($wopt_clear);
}
// Contents
//////////////////////////////////////////
if ($p['contents']) {
// should be rewritten to check against filemtimes
echo ProgressBar::start(count($fx),"Gathering contents (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$cpath = $bpath."/contents/".substr($fid, 0, 2);
$cfile = $cpath."/".$fid.".zip";
if (in_array($ext, $p['c_files'])) {
if (!is_dir($cpath)) { mkdir($cpath); }
$max_size = $p['contents_limit'] * 1000;
if ($ext == ".DS_Store") {
// store all DS_Store files no matter how big
$my_size = 1;
} else {
$my_size = filesize($pathname);
}
if (!file_exists($cfile) && $my_size < $max_size) {
msg("Zipping ".$pathname);
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
$zip->addfile($pathname,basename($pathname));
$zip->close();
$stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($cfile, strlen($bpath)));
$stmt->execute();
}
}
echo ProgressBar::next(true);
}
echo ProgressBar::finish($wopt_clear);
}
// Pool Indices // Pool Indices
////////////////////////////////////////// //////////////////////////////////////////

8
YubaPrefs.php
View File

@@ -13,6 +13,14 @@ function makeWindowString($p, $strings) {
*.title = Preferences *.title = Preferences
*.floating = 1 *.floating = 1
parallel.type = checkbox
parallel.label = Execute shell commands in parallel
parallel.default = ".$p['parallel']."
debug.type = checkbox
debug.label = Enable verbose logging
debug.default = ".$p['debug']."
bdest.type = openbrowser bdest.type = openbrowser
bdest.filetype = directory bdest.filetype = directory
bdest.label = Destination bdest.label = Destination

2
current_version.txt
View File

@@ -1 +1 @@
0.7.13.1 0.7.13.2

18
functions.php
View File

@@ -76,6 +76,13 @@ class ProgressBar {
// Functions // Functions
////////////////////////////////////////// //////////////////////////////////////////
function reviseprefs($array) { // bad practice
$prefs_file = "/Users/".get_current_user()."/Library/Preferences/org.profiteroles.Yuba.php";
$p = unserialize(file_get_contents($prefs_file));
foreach ($array as $key => $value) { $p[$key] = "value"; }
file_put_contents($prefs_file,serialize($result));
}
function prettysize($size) { function prettysize($size) {
$unit=array('b','kb','mb','gb','tb','pb'); $unit=array('b','kb','mb','gb','tb','pb');
return @round($size/pow(1024,($i=floor(log($size,1024)))),2).' '.$unit[$i]; return @round($size/pow(1024,($i=floor(log($size,1024)))),2).' '.$unit[$i];
@@ -99,9 +106,20 @@ function msg($string) {
global $messages_log_file; global $messages_log_file;
$logstring = "[".date('Y-m-d h:i:s')."] ".$string."\n"; $logstring = "[".date('Y-m-d h:i:s')."] ".$string."\n";
file_put_contents($messages_log_file, $logstring, FILE_APPEND); file_put_contents($messages_log_file, $logstring, FILE_APPEND);
file_put_contents("/tmp/yuba/debug.log", $logstring, FILE_APPEND);
return $string."\n"; return $string."\n";
} }
function dm($string) {
global $p; if (!$p['debug']) { return; } // bad practice
if (strpos($string,"\n") === false) {
$logstring = "[".date('Y-m-d h:i:s')."] ".$string."\n";
} else {
$logstring = "[".date('Y-m-d h:i:s')."]\n".$string."\n".str_repeat("-",33)."\n";
}
file_put_contents("/tmp/yuba/debug.log", $logstring, FILE_APPEND);
}
function timeToSeconds($val) { function timeToSeconds($val) {
if (!is_numeric($val) && strpos($val,":") === false) { if (!is_numeric($val) && strpos($val,":") === false) {
$val = str_replace(" s","",$val); $val = str_replace(" s","",$val);

444
helper.php
View File

@@ -10,31 +10,189 @@ $p = unserialize(file_get_contents($prefs_file));
require("functions.php"); require("functions.php");
require("filetypes.php"); require("filetypes.php");
$mode = $argv[1]; $dm = "";
$fid = $argv[2]; function dfm ($string, $flag = 0) {
$pathname = $argv[3]; // dumb workaround for debug messages appearing out of order
$bpath = $argv[4]; global $dm;
$mytime = $argv[5]; if ($flag) {
$dm .= str_repeat("-",33)."\n".$string."\n".str_repeat("-",33)."\n";
} else {
$dm .= "[".date('Y-m-d h:i:s')."] ".$string."\n";
}
}
$fid = $argv[1];
$pathname = $argv[2];
$shellpath = escapeshellarg($pathname);
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$bpath = $argv[3];
$mytime = $argv[4];
$stamp = date("Y-m-d_H-i-s", $mytime);
$messages_log_file = $bpath."/".$stamp."_messages.log";
$tmpdir = "/tmp/yuba/".$mytime; $tmpdir = "/tmp/yuba/".$mytime;
if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); } if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); dfm("mkdir ",$tmpdir); }
$file = pathinfo($pathname, PATHINFO_BASENAME); $file = pathinfo($pathname, PATHINFO_BASENAME);
$ext = pathinfo($pathname, PATHINFO_EXTENSION); $ext = pathinfo($pathname, PATHINFO_EXTENSION);
$shellpath = escapeshellarg($pathname);
$dbp = new PDO("sqlite:".$bpath."/pool.sqlite3"); $dbp = new PDO("sqlite:".$bpath."/pool.sqlite3");
$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbp->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); $dbp->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
echo "\nBatch (".$mode."): ".basename($pathname); //$dbp->query("PRAGMA page_size = 4096");
//$dbp->query("PRAGMA cache_size = 10000");
//$dbp->query("PRAGMA locking_mode = EXCLUSIVE");
//$dbp->query("PRAGMA synchronous = NORMAL");
$dbp->query("PRAGMA journal_mode = WAL");
switch ($mode) { $sbatch = array();
// Icons ////////////////////////////////////////// $estring = "\nBATCH: ".basename($pathname);
case "icons": dfm("BATCH INIT ".$file.":".print_r($argv,true));
// Thumbs //////////////////////////////////////////
thumbs:
if ($p['thumbs']) {
dfm("THUMBS",1);
$estring .= " THUMBS";
// "bad" filesizes
$discard = array( 3953,
4977,
5019,
6059,
6616,
17393 );
$tfile = $tmpdir."/".substr($fid,0,2)."/".$fid.".jpg";
$tpfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention
$dfile = $bpath."/thumbs/".substr($fid, 0, 2)."/".$fid.".jpg";;
if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); dfm("mkdir ".dirname($tfile)); }
if (in_array($ext, $p['t_skip'])) {
// file extension is in the skip list
$estring .= " ->t_skip";
dfm("file extension is in the skip list");
goto icons;
}
if ($p['thumbs'] == 1 && file_exists($dfile)) {
// if this is not a rebuild, first check for an existing thumb file (faster)
dfm("this is not a rebuild, and an existing thumb file was found");
$estring .= " ->skip_fast";
goto icons;
}
$rowid = @$dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()['rowid'];
// check for existing thumb generation attempt
if (!$rowid) {
// no prior attempt
dfm("no prior attempt at thumb file generation was made for ".$fid);
$stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)");
dfm("mysql insert for thumb generation attempt");
$estring .= " ->generate";
} elseif ($rowid && $p['thumbs'] == 2) {
// prior attempt but rebuild mode
$stmt = $dbp->prepare("UPDATE thumbs SET fid = :fid, created = :created, relative_path = :relative_path, width = :width, height = :height, tool = :tool WHERE rowid = ".$rowid);
dfm("mysql update for rebuild");
$estring .= " ->rebuild";
} else {
// prior attempt
dfm("a prior attempt at thumb generation was found for ".$fid);
$estring .= " ->skip";
goto icons;
}
$cmd['sips'] = $bin_sips." -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; // add ?? "-d profile --deleteColorManagementProperties"
$cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size'];
//$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tfile;
$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile;
$cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile;
$cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8";
$cmd['qlmanage'] = $bin_qlmanage." -t -s ".$p['thumb_size']." -o ".dirname($tpfile)." ".$shellpath."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile;
if (in_array($ext, $p['t_files']['sips'])) {
$external_tool = "sips";
} elseif (in_array($ext, $p['t_files']['ffmpeg'])) {
$external_tool = "ffmpeg";
} elseif (in_array($ext, $p['t_files']['sox'])) {
$external_tool = "sox";
} else {
$external_tool = null;
}
dfm("external tool for ".$ext." = ".$external_tool);
switch ($p['thumb_priority']) {
// external tool priority
case 0: $priority = array($external_tool,"ql-thumbnail","qlmanage"); break;
// ql-thumbnail priority
case 1: $priority = array("ql-thumbnail","qlmanage",$external_tool); break;
// qlmanage priority
case 2: $priority = array("qlmanage","ql-thumbnail",$external_tool); break;
}
dfm("priority for ".$ext." is: ".print_r($priority,true));
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); dfm("mkdir ".dirname($dfile)); }
foreach ($priority as $tool) {
if (empty($cmd[$tool])) { break; }
shell_exec($cmd[$tool]." 2>&1");
dfm($cmd[$tool]);
$estring .= " ->".$tool;
$checksize = @filesize($tfile);
if ($checksize && !in_array($checksize,$discard)) {
$estring .= " ->use";
$stmt->BindValue(":tool",$tool);
$stmt->BindValue(":relative_path",substr($dfile, strlen($bpath)));
list($width, $height) = getimagesize($tfile);
$stmt->BindValue(":width",$width);
$stmt->BindValue(":height",$height);
rename($tfile,$dfile);
dfm("moving ".$tfile." to ".$dfile);
break;
} else {
$estring .= " ->discard";
dfm("filesize ".$checksize." is in the banned size array (or is zero)");
}
}
$sbatch[] = $stmt;
$estring .= " ->db";
}
// Icons //////////////////////////////////////////
icons:
if ($p['icons']) {
dfm("ICONS",1);
$estring .= " ICONS";
if ($p['icon_tool'] == 0) { if ($p['icon_tool'] == 0) {
$tool = "qltool"; $tool = "qltool";
@@ -44,8 +202,8 @@ switch ($mode) {
if (in_array($ext, $p['i_skip'])) { if (in_array($ext, $p['i_skip'])) {
// file extension is in the skip list // file extension is in the skip list
echo " ->i_skip"; $estring .= " ->i_skip";
break; goto meta;
} }
$tfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; $tfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png";
@@ -59,19 +217,19 @@ switch ($mode) {
// no prior attempt // no prior attempt
$stmt = $dbp->prepare("INSERT INTO icons VALUES (:fid, :hash, :created, :relative_path, :tool)"); $stmt = $dbp->prepare("INSERT INTO icons VALUES (:fid, :hash, :created, :relative_path, :tool)");
echo " ->generate"; $estring .= " ->generate";
} elseif ($rowid && $p['icons'] == 2) { } elseif ($rowid && $p['icons'] == 2) {
// prior attempt but rebuild mode // prior attempt but rebuild mode
$stmt = $dbp->prepare("UPDATE icons SET fid = :fid, hash = :hash, created = :created, relative_path = :relative_path, tool = :tool WHERE rowid = ".$rowid); $stmt = $dbp->prepare("UPDATE icons SET fid = :fid, hash = :hash, created = :created, relative_path = :relative_path, tool = :tool WHERE rowid = ".$rowid);
echo " ->rebuild"; $estring .= " ->rebuild";
} else { } else {
// prior attempt // prior attempt
echo " ->skip"; $estring .= " ->skip";
break; goto meta;
} }
@@ -104,14 +262,14 @@ switch ($mode) {
$stmt->BindValue(":relative_path",$row['relative_path']); $stmt->BindValue(":relative_path",$row['relative_path']);
$stmt->BindValue(":tool",$row['tool']); $stmt->BindValue(":tool",$row['tool']);
echo " ->recycle"; $estring .= " ->recycle";
} else { } else {
$dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png"; $dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png";
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); } if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); }
echo " ->use"; $estring .= " ->use";
if ($tool == "qltool") { if ($tool == "qltool") {
// qltool makes 2x icons for some reason // qltool makes 2x icons for some reason
@@ -134,132 +292,138 @@ switch ($mode) {
} else { } else {
echo " ->discard"; $estring .= " ->discard";
} }
$stmt->execute(); $sbatch[] = $stmt;
echo " ->db"; $estring .= " ->db";
break;
case "thumbs":
// Thumbs //////////////////////////////////////////
// "bad" filesizes
$discard = array( 3953,
4977,
5019,
6059,
6616,
17393 );
$tfile = $tmpdir."/".substr($fid,0,2)."/".$fid.".jpg";
$tpfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention
if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); }
if (in_array($ext, $p['t_skip'])) {
// file extension is in the skip list
echo " ->t_skip";
break;
}
if ($p['thumbs'] == 1 && file_exists($tfile)) {
// if this is not a rebuild, first check for an existing thumb file (faster)
echo " ->skip_fast";
break;
}
$rowid = @$dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()['rowid'];
// check for existing thumb generation attempt
if (!$rowid) {
// no prior attempt
$stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)");
echo " ->generate";
} elseif ($rowid && $p['thumbs'] == 2) {
// prior attempt but rebuild mode
$stmt = $dbp->prepare("UPDATE thumbs SET fid = :fid, created = :created, relative_path = :relative_path, width = :width, height = :height, tool = :tool WHERE rowid = ".$rowid);
echo " ->rebuild";
} else {
// prior attempt
echo " ->skip";
break;
}
$cmd['sips'] = $bin_sips." -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; // add ?? "-d profile --deleteColorManagementProperties"
$cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size'];
//$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tfile;
$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile;
$cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile;
$cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8";
$cmd['qlmanage'] = $bin_qlmanage." -t -s ".$p['thumb_size']." -o ".dirname($tpfile)." ".$shellpath."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile;
if (in_array($ext, $p['t_files']['sips'])) {
$external_tool = "sips";
} elseif (in_array($ext, $p['t_files']['ffmpeg'])) {
$external_tool = "ffmpeg";
} elseif (in_array($ext, $p['t_files']['sox'])) {
$external_tool = "sox";
} else {
$external_tool = null;
}
switch ($p['thumb_priority']) {
// external tool priority
case 0: $priority = array($external_tool,"ql-thumbnail","qlmanage"); break;
// ql-thumbnail priority
case 1: $priority = array("ql-thumbnail","qlmanage",$external_tool); break;
// qlmanage priority
case 2: $priority = array("qlmanage","ql-thumbnail",$external_tool); break;
}
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$dfile = $bpath."/thumbs/".substr($fid, 0, 2)."/".$fid.".jpg";;
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); }
foreach ($priority as $tool) {
if (empty($cmd[$tool])) { continue; }
shell_exec($cmd[$tool]." 2>&1");
echo " ->".$tool;
$checksize = @filesize($tfile);
if ($checksize && !in_array($checksize,$discard)) {
echo " ->use";
$stmt->BindValue(":tool",$tool);
$stmt->BindValue(":relative_path",substr($dfile, strlen($bpath)));
list($width, $height) = getimagesize($tfile);
$stmt->BindValue(":width",$width);
$stmt->BindValue(":height",$height);
rename($tfile,$dfile);
break;
} else {
echo " ->discard";
}
}
$stmt->execute();
echo " ->db";
break;
} }
echo "\n"; // Metadata //////////////////////////////////////////
meta:
if ($p['meta']) {
dfm("META",1);
$estring .= " META";
if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) {
$estring .= " ->notmedia";
goto hashy;
}
if (in_array($ext, $p['e_files'])) {
$check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch();
if (!reset($check)) { // hacky but needed because we set FETCH_ASSOC
$arrstring = shell_exec($bin_exiftool." -php ".$shellpath);
// $rawexif = eval("return ".`$bin_exiftool -php $shellpath`);
// do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'"
if (substr($arrstring,0,5) == "Array") {
$rawexif = eval("return ".$arrstring);
$stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":tags",serialize($rawexif[0]));
$sbatch[] = $stmt;
$estring .= " ->etool";
}
} else {
$estring .= " ->e_found";
}
}
if (in_array($ext, $p['m_files'])) {
$check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch();
if (!reset($check)) {
$stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)");
$stmt->BindValue(":fid",$fid);
//$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"))));
//$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"));
$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1"));
$sbatch[] = $stmt;
$estring .= " ->minfo";
} else {
$estring .= " ->m_found";
}
}
}
// Hash //////////////////////////////////////////
hashy:
if ($p['hash']) {
dfm("HASH",1);
$estring .= " HASH";
$size = filesize($pathname);
$limit = $p['hash_limit']*1000000000;
$check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch();
if (reset($check)) {
$estring .= " ->exists";
} elseif ($p['hash_limit'] && ($size > $limit)) {
$estring .= " ->too big";
} else {
$estring .= " ->generating";
$stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":hash",md5_file($pathname));
$sbatch[] = $stmt;
}
$estring .= " ->done";
}
// Contents //////////////////////////////////////////
contents:
if ($p['contents']) {
dfm("CONTENTS",1);
$estring .= " CONTENTS";
$cpath = $bpath."/contents/".substr($fid, 0, 2);
$cfile = $cpath."/".$fid.".zip";
if (in_array($ext, $p['c_files'])) {
if (!is_dir($cpath)) { mkdir($cpath); }
$max_size = $p['contents_limit'] * 1000;
if ($ext == ".DS_Store") {
// store all DS_Store files no matter how big
$my_size = 1;
} else {
$my_size = filesize($pathname);
}
if (!file_exists($cfile) && $my_size < $max_size) {
$estring .= " ->zip";
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
$zip->addfile($pathname,basename($pathname));
$zip->close();
$stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($cfile, strlen($bpath)));
$sbatch[] = $stmt;
}
}
}
// Inserts //////////////////////////////////////////
foreach ($sbatch as $stmt) { $stmt->execute(); }
dm($dm);
echo msg($estring);
?> ?>

2
prefs.php
View File

@@ -1 +1 @@
a:18:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"stat_mode";i:1;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:14:"thumb_priority";i:0;s:5:"icons";i:1;s:9:"icon_size";i:512;s:9:"icon_tool";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;} a:20:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"stat_mode";i:1;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:14:"thumb_priority";i:0;s:5:"icons";i:1;s:9:"icon_size";i:512;s:9:"icon_tool";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;s:8:"parallel";i:1;s:5:"debug";i:0;}