p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.8.0.0

Browse Source
This commit is contained in:
profiteroles
2019-10-15 18:32:21 -07:00
parent 1bf543ec00
commit a344259e40
77 changed files with 308 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

292
helper.php
View File

@@ -13,9 +13,14 @@ require("filetypes.php");
$dm = "";
function dfm ($string, $flag = 0) {
// dumb workaround for debug messages appearing out of order
// bypass for single-thread to observe hangs
//dm($string);
//return
global $dm;
if ($flag) {
if ($flag == 1) {
$dm .= str_repeat("-",33)."\n".$string."\n".str_repeat("-",33)."\n";
} elseif ($flag == 2) {
$dm .= str_repeat("-",66)."\n".$string."\n".str_repeat("-",66)."\n";
} else {
$dm .= "[".date('Y-m-d h:i:s')."] ".$string."\n";
}
@@ -25,6 +30,7 @@ $fid = $argv[1];
$pathname = $argv[2];
$shellpath = escapeshellarg($pathname);
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$file = pathinfo($pathname, PATHINFO_BASENAME);
$bpath = $argv[3];
$mytime = $argv[4];
@@ -34,9 +40,6 @@ $messages_log_file = $bpath."/".$stamp."_messages.log";
$tmpdir = "/tmp/yuba/".$mytime;
if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); dfm("mkdir ",$tmpdir); }
$file = pathinfo($pathname, PATHINFO_BASENAME);
$ext = pathinfo($pathname, PATHINFO_EXTENSION);
$dbp = new PDO("sqlite:".$bpath."/pool.sqlite3");
$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbp->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
@@ -49,9 +52,9 @@ $dbp->query("PRAGMA journal_mode = WAL");
$sbatch = array();
$estring = "\nBATCH: ".basename($pathname);
$estring = "\nBATCH: ".shortlabel(basename($pathname),20);
dfm("BATCH INIT ".$file.":".print_r($argv,true));
dfm("BATCH INIT ".$file.":".print_r($argv,true),2);
// Thumbs //////////////////////////////////////////
@@ -62,7 +65,7 @@ if ($p['thumbs']) {
$estring .= " THUMBS";
// "bad" filesizes
// "bad" filesizes from generic icons
$discard = array( 3953,
4977,
5019,
@@ -70,13 +73,13 @@ if ($p['thumbs']) {
6616,
17393 );
$tfile = $tmpdir."/".substr($fid,0,2)."/".$fid.".jpg";
$tpfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention
$tfile = $tmpdir."/thumbs/".substr($fid,0,2)."/".$fid.".jpg";
$tpfile = $tmpdir."/thumbs/".substr($fid,0,2)."/".$file.".png"; // workaround for qlmanage naming convention
$dfile = $bpath."/thumbs/".substr($fid, 0, 2)."/".$fid.".jpg";;
if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); dfm("mkdir ".dirname($tfile)); }
if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile),0777,true); dfm("mkdir ".dirname($tfile)); }
if (in_array($ext, $p['t_skip'])) {
if ($p['thumb_mode'] != 0 && in_array($ext, $p['t_skip'])) {
// file extension is in the skip list
$estring .= " ->t_skip";
dfm("file extension is in the skip list");
@@ -89,10 +92,11 @@ if ($p['thumbs']) {
$estring .= " ->skip_fast";
goto icons;
}
$rowid = @$dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()['rowid'];
// check for existing thumb generation attempt
$sql = "SELECT rowid FROM thumbs WHERE fid='".$fid."'";
dfm($sql);
$rowid = @$dbp->query($sql)->fetch()['rowid'];
if (!$rowid) {
// no prior attempt
@@ -113,7 +117,7 @@ if ($p['thumbs']) {
// prior attempt
dfm("a prior attempt at thumb generation was found for ".$fid);
$estring .= " ->skip";
$estring .= " ->pskip";
goto icons;
}
@@ -121,8 +125,7 @@ if ($p['thumbs']) {
$cmd['sips'] = $bin_sips." -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile; // add ?? "-d profile --deleteColorManagementProperties"
$cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size'];
//$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o - | ".$bin_convert." - -crop 800x515+58+30 -scale 515x515! +dither -colors 16 ".$tfile;
$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/10}') spectrogram -o ".$tpfile."; ".$bin_sips." -s format jpeg -s formatOptions 80 ".escapeshellarg($tpfile)." --out ".$tfile;
$cmd['sox'] = $bin_sox." ".$shellpath." -n trim 0 $(".$bin_exiftool." -s -s -s -duration# ".$shellpath." | awk '{print $1/4}') spectrogram -r -o - | ".$bin_convert." - -quality 85% ".$tfile;
$cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile;
$cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8";
@@ -138,43 +141,47 @@ if ($p['thumbs']) {
$external_tool = null;
}
dfm("external tool for ".$ext." = ".$external_tool);
switch ($p['thumb_priority']) {
// external tool priority
switch ($p['thumb_mode']) {
case 0: $priority = array($external_tool,"ql-thumbnail","qlmanage"); break;
// ql-thumbnail priority
case 1: $priority = array("ql-thumbnail","qlmanage",$external_tool); break;
// qlmanage priority
case 2: $priority = array("qlmanage","ql-thumbnail",$external_tool); break;
case 1: $priority = array("ql-thumbnail","qlmanage"); break;
case 2: $priority = array("ql-thumbnail"); break;
}
dfm("thumb mode ".$p['thumb_mode']);
dfm("priority for ".$ext." is: ".print_r($priority,true));
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); dfm("mkdir ".dirname($dfile)); }
foreach ($priority as $tool) {
if (empty($cmd[$tool])) { break; }
if (empty($cmd[$tool])) { continue; }
shell_exec($cmd[$tool]." 2>&1");
dfm($cmd[$tool]);
shell_exec($cmd[$tool]." 2>&1");
$estring .= " ->".$tool;
$checksize = @filesize($tfile);
if ($checksize && !in_array($checksize,$discard)) {
$estring .= " ->use";
$stmt->BindValue(":tool",$tool);
$stmt->BindValue(":relative_path",substr($dfile, strlen($bpath)));
list($width, $height) = getimagesize($tfile);
$stmt->BindValue(":width",$width);
$stmt->BindValue(":height",$height);
rename($tfile,$dfile);
dfm("moving ".$tfile." to ".$dfile);
break;
if ($checksize) {
if (!empty($discard) && in_array($checksize,$discard)) {
$estring .= " ->discard";
dfm($checksize." is in the banned size array");
} else {
$estring .= " ->use";
dfm ("generated ".$tfile." filesize is ".$checksize);
$stmt->BindValue(":tool",$tool);
$stmt->BindValue(":relative_path",substr($dfile, strlen($bpath)));
dfm("getimagesize on ".$tfile);
list($width, $height) = getimagesize($tfile);
$stmt->BindValue(":width",$width);
$stmt->BindValue(":height",$height);
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); dfm("mkdir ".dirname($dfile)); }
rename($tfile,$dfile);
dfm("moving ".$tfile." to ".$dfile);
break;
}
} else {
$estring .= " ->discard";
dfm("filesize ".$checksize." is in the banned size array (or is zero)");
dfm($tool." produced a bad file (size=".$checksize.")");
}
}
@@ -193,109 +200,129 @@ if ($p['icons']) {
dfm("ICONS",1);
$estring .= " ICONS";
if ($p['icon_tool'] == 0) {
$tool = "qltool";
} elseif ($p['icon_tool'] == 1) {
$tool = "qlmanage";
}
if (in_array($ext, $p['i_skip'])) {
// file extension is in the skip list
if ($p['icon_mode'] != 0 && in_array($ext, $p['i_skip'])) {
$estring .= " ->i_skip";
dfm("file extension is in the skip list");
goto meta;
}
$tfile = $tmpdir."/".substr($fid,0,2)."/".$file.".png";
if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile)); }
// workaround for qlmanage naming convention
$tfile = $tmpdir."/icons/".substr($fid,0,2)."/".$fid.".png";
if (!is_dir(dirname($tfile))) { @mkdir(dirname($tfile),0777,true); dfm("mkdir ".dirname($tfile)); }
$rowid = @$dbp->query("SELECT rowid FROM icons WHERE fid='".$fid."'")->fetch()['rowid'];
// check for existing icon generation attempt
// we can't check for files (faster) here because they are stored by md5_file hash
$sql = "SELECT rowid FROM icons WHERE fid='".$fid."'";
dfm($sql);
$rowid = @$dbp->query($sql)->fetch()['rowid'];
if (!$rowid) {
// no prior attempt
dfm("no prior attempt was found");
$stmt = $dbp->prepare("INSERT INTO icons VALUES (:fid, :hash, :created, :relative_path, :tool)");
$estring .= " ->generate";
} elseif ($rowid && $p['icons'] == 2) {
// prior attempt but rebuild mode
dfm("a prior attempt was found but we are in rebuild mode");
$stmt = $dbp->prepare("UPDATE icons SET fid = :fid, hash = :hash, created = :created, relative_path = :relative_path, tool = :tool WHERE rowid = ".$rowid);
$estring .= " ->rebuild";
} else {
// prior attempt
$estring .= " ->skip";
dfm("a prior attempt was found, skipping");
$estring .= " ->pskip";
goto meta;
}
$stmt->BindValue(":fid",$fid);
if ($tool == "qltool") {
$cmd = $bin_qltool." di ".$shellpath." ".$p['icon_size']." ".$p['icon_size']." | base64 --decode > ".escapeshellarg($tfile);
} elseif ($tool == "qlmanage") {
// > code for custom qlgenerator bindings goes here <
$cmd = $bin_qlmanage." -ti -s ".$p['icon_size']." -o ".dirname($tfile)." ".$shellpath;
//$cmd = $bin_qlmanage." -ti -f ".floor($p['icon_size']/128)." -o ".dirname($tfile)." ".$shellpath;
}
shell_exec($cmd);
if (@filesize($tfile)) {
$hash = md5_file($tfile);
$stmt->BindValue(":hash",$hash);
if ($row = @$dbp->query("SELECT * FROM icons WHERE hash='".$hash."'")->fetchAll()[0]) {
// check for another icon with the same hash
$stmt->BindValue(":created",$row['created']);
$stmt->BindValue(":relative_path",$row['relative_path']);
$stmt->BindValue(":tool",$row['tool']);
$estring .= " ->recycle";
} else {
$dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png";
$cmd['flacicon'] = $bin_flacicon." ".$shellpath." ".$p['icon_size']." ".$tfile;
$cmd['ql-icon'] = $bin_qlicon." --input=".$shellpath." --width=".$p['icon_size']." --height=".$p['icon_size']." | ".$bin_pngquant." - -o ".$tfile;
$cmd['qltool'] = $bin_qltool." di ".$shellpath." ".$p['icon_size']." ".$p['icon_size']." | base64 --decode | ".$bin_convert." - -scale 50% - | ".$bin_pngquant." - -o ".$tfile;
if ($p['icon_mode'] == 2) {
dfm("icon mode 2, bypassing hash check");
$tool = "ql-icon";
shell_exec($cmd[$tool]." 2>&1");
$stmt->BindValue(":tool",$tool);
dfm($cmd[$tool]);
$estring .= " ->".$tool;
$checksize = @filesize($tfile);
if ($checksize) {
dfm("generated ".$tfile." filesize is ".$checksize);
$dfile = $bpath."/icons/".substr($fid, 0, 2)."/".$fid.".png";
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); }
rename($tfile,$dfile);
dfm("moving ".$tfile." to ".$dfile);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($dfile,strlen($bpath)));
$stmt->BindValue(":hash",null);
$estring .= " ->use";
} else {
$estring .= " ->discard";
dfm($tool." produced a bad file (size=".$checksize.")");
}
} else {
if ($ext == "flac") {
$tools = array("flacicon","ql-icon","qltool");
} else {
$tools = array("ql-icon","qltool");
}
foreach ($tools as $tool) {
if (!@filesize($tfile)) {
shell_exec($cmd[$tool]." 2>&1");
dfm($cmd[$tool]);
$estring .= " ->".$tool;
}
}
$checksize = @filesize($tfile);
if ($checksize) {
dfm ("generated ".$tfile." filesize is ".$checksize);
dfm("md5_file on ".$tfile);
$hash = md5_file($tfile);
$stmt->BindValue(":hash",$hash);
$sql = "SELECT * FROM icons WHERE hash='".$hash."'";
dfm($sql);
$row = @$dbp->query($sql)->fetchAll()[0];
if ($tool == "qltool") {
// qltool makes 2x icons for some reason
//shell_exec($bin_sips." -z ".$p['icon_size']." ".$p['icon_size']." ".$tfile." --out ".$dfile);
shell_exec($bin_convert." ".escapeshellarg($tfile)." -scale 50% -strip -define png:compression-level=9 ".$dfile);
if ($row) {
dfm("found another record with the same hash");
$estring .= " ->recycle";
$stmt->BindValue(":created",$row['created']);
$stmt->BindValue(":relative_path",$row['relative_path']);
$stmt->BindValue(":tool",$row['tool']);
} else {
//rename($tfile,$dfile);
shell_exec($bin_convert." ".escapeshellarg($tfile)." -strip -define png:compression-level=9 ".$dfile);
dfm("no existing hash record");
$dfile = $bpath."/icons/".substr($hash, 0, 2)."/".$hash.".png";
if (!is_dir(dirname($dfile))) { @mkdir(dirname($dfile)); }
rename($tfile,$dfile);
dfm("moving ".$tfile." to ".$dfile);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($dfile,strlen($bpath)));
$stmt->BindValue(":tool",$tool);
$estring .= " ->use";
}
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($dfile,strlen($bpath)));
$stmt->BindValue(":tool",$tool);
}
} else {
$estring .= " ->discard";
}
} else {
$estring .= " ->discard";
dfm($tool." produced a bad file (size=".$checksize.")");
}
}
$sbatch[] = $stmt;
$estring .= " ->db";
@@ -313,15 +340,19 @@ if ($p['meta']) {
if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) {
$estring .= " ->notmedia";
dfm("file extension ".$ext." is not a media file");
goto hashy;
}
if (in_array($ext, $p['e_files'])) {
$check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch();
if (!reset($check)) { // hacky but needed because we set FETCH_ASSOC
$arrstring = shell_exec($bin_exiftool." -php ".$shellpath);
// $rawexif = eval("return ".`$bin_exiftool -php $shellpath`);
// do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'"
$sql = "SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')";
dfm($sql);
$check = @$dbp->query($sql)->fetch();
if (!reset($check)) {
dfm("creating exiftool dump");
$cmd = $bin_exiftool." -php ".$shellpath;
dfm($cmd);
$arrstring = shell_exec($cmd);
if (substr($arrstring,0,5) == "Array") {
$rawexif = eval("return ".$arrstring);
$stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)");
@@ -332,21 +363,24 @@ if ($p['meta']) {
}
} else {
$estring .= " ->e_found";
dfm("existing exiftool dump found");
}
}
if (in_array($ext, $p['m_files'])) {
$check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch();
$sql = "SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')";
dfm($sql);
$check = @$dbp->query($sql)->fetch();
if (!reset($check)) {
dfm("creating mediainfo dump");
$stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)");
$stmt->BindValue(":fid",$fid);
//$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"))));
//$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"));
$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1"));
$sbatch[] = $stmt;
$estring .= " ->minfo";
} else {
$estring .= " ->m_found";
dfm("existing mediainfo dump found");
}
}
@@ -362,14 +396,20 @@ if ($p['hash']) {
$estring .= " HASH";
$size = filesize($pathname);
dfm("file ".$pathname." is ".$size);
$limit = $p['hash_limit']*1000000000;
$check = @$dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch();
$sql = "SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')";
dfm($sql);
$check = @$dbp->query($sql)->fetch();
if (reset($check)) {
$estring .= " ->exists";
dfm("hash already exists for ".$pathname);
} elseif ($p['hash_limit'] && ($size > $limit)) {
$estring .= " ->too big";
dfm("file is too big to hash");
} else {
$estring .= " ->generating";
dfm("generating hash for ".$pathname);
$stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":hash",md5_file($pathname));
@@ -392,17 +432,21 @@ if ($p['contents']) {
$cpath = $bpath."/contents/".substr($fid, 0, 2);
$cfile = $cpath."/".$fid.".zip";
if (in_array($ext, $p['c_files'])) {
if (!is_dir($cpath)) { mkdir($cpath); }
$max_size = $p['contents_limit'] * 1000;
if ($ext == ".DS_Store") {
// store all DS_Store files no matter how big
dfm("flagging ds_store file for inclusion");
$my_size = 1;
} else {
$my_size = filesize($pathname);
}
dfm("file ".$pathname." is ".$my_size);
}
if (!file_exists($cfile) && $my_size < $max_size) {
if (!is_dir($cpath)) { mkdir($cpath); dfm("mkdir ",$cpath); }
$estring .= " ->zip";
dfm("creating a zip archive of ".$pathname);
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
@@ -421,9 +465,11 @@ if ($p['contents']) {
// Inserts //////////////////////////////////////////
foreach ($sbatch as $stmt) { $stmt->execute(); }
dfm("SQL",1);
foreach ($sbatch as $stmt) { $stmt->execute(); dfm($stmt->queryString); }
dm($dm);
echo msg($estring);
echo msg($estring,false);
?>