p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.7.13.2

Browse Source
This commit is contained in:
profiteroles
2019-10-09 04:07:46 -07:00
parent 197c100c01
commit e325062915
14 changed files with 458 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
Yuba.php
View File

@@ -7,6 +7,7 @@ $version = file_get_contents(__DIR__."/current_version.txt");
ini_set('memory_limit', '10240M');
date_default_timezone_set("America/Los_Angeles");
if (!file_exists("/tmp/yuba/debug.log")) { touch("/tmp/yuba/debug.log"); }
// Includes & Prefs
//////////////////////////////////////////
@@ -25,10 +26,9 @@ require("filetypes.php");
// Manual prefs
$wopt_steps = 12; // total number of steps
$wopt_steps = 8; // total number of steps
$wopt_currstep = 1;
$wopt_clear = 0; // set to 0 for debug
$wopt_parallel = 1; // use parallel?
if ($p['debug']) { $wopt_clear = 0; } else { $wopt_clear = 1; }
$parser = new plistParser();
@@ -42,6 +42,13 @@ if (@$argv[1] == "Preferences...") {
die;
}
// Console
if (@$argv[1] == "Console") {
exec("open -n ".__DIR__."/bin/Console.app --args /tmp/yuba/debug.log");
die;
}
// Version check
if (@$argv[1] == "Check for Updates...") {
@@ -59,13 +66,29 @@ if (@$argv[1] == "Check for Updates...") {
}
dm("Launching Yuba\n".str_repeat("-",33)."\n".print_r($p,true));
// PHP Checks
//////////////////////////////////////////
$needed = array("iconv","fileinfo","json","PDO","pdo_sqlite","SimpleXML","sqlite3","xml");
foreach ($needed as $ext) {
if (!extension_loaded($ext)) {
alert("PHP is missing the ".$ext.". Exiting.","PHP Extension Missing");
echo "QUITAPP\n";
}
}
if ($p['contents'] && !extension_loaded("zip")) {
alert("PHP is missing the zip extension. Yuba will not collect file contents.","PHP Extension Missing");
revise_prefs(array("contents" => 0));
}
// Path & application variables
//////////////////////////////////////////
$mytime = time();
$tmpdir = "/tmp/yuba/".$mytime;
if (!is_dir($tmpdir)) { mkdir($tmpdir,0777,true); }
$stamp = date("Y-m-d_H-i-s", $mytime);
if (!isset($argv[1]) || $argv[1] == "") { echo "No input"; die; }
@@ -199,10 +222,10 @@ $dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3");
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$dbo->query("PRAGMA page_size = 4096");
$dbo->query("PRAGMA cache_size = 10000");
$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
$dbo->query("PRAGMA synchronous = NORMAL");
//$dbo->query("PRAGMA page_size = 4096");
//$dbo->query("PRAGMA cache_size = 10000");
//$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
//$dbo->query("PRAGMA synchronous = NORMAL");
$dbo->query("PRAGMA journal_mode = WAL");
$dbo->exec("CREATE TABLE _skim (
@@ -376,12 +399,6 @@ $dbp = new PDO("sqlite:".$bpath."/pool.sqlite3");
$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
//$dbp->query("PRAGMA page_size = 4096");
//$dbp->query("PRAGMA cache_size = 10000");
//$dbp->query("PRAGMA locking_mode = EXCLUSIVE");
//$dbp->query("PRAGMA synchronous = NORMAL");
//$dbp->query("PRAGMA journal_mode = WAL");
$dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)");
@@ -554,29 +571,29 @@ foreach ($fx as $array) {
}
echo ProgressBar::finish($wopt_clear);
// Thumbs
// Helper
//////////////////////////////////////////
if ($p['thumbs']) {
if ($p['thumbs'] || $p['icons'] || $p['meta'] || $p['hash'] || $p['contents']) {
echo ProgressBar::start($passed_file,"Creating thumb batch (".stepString().")",1);
echo ProgressBar::start($passed_file,"Creating batch (".stepString().")",1);
$batchfile = $tmpdir."/_batch_thumbs.sh";
$batchfile = $tmpdir."/_batch.sh";
$helper = realpath("helper.php");
foreach ($fx as $count => $array) {
$fid = $array[0];
$pathname = $array[1];
$tcmd = $bin_php." ".escapeshellarg($helper)." thumbs ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next();
$tcmd = $bin_php." ".escapeshellarg($helper)." ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; printf '\\n".ProgressBar::next()."'";
msg($tcmd);
$tline[] = $tcmd;
$line[] = $tcmd;
}
file_put_contents($batchfile,implode("\n", $tline));
file_put_contents($batchfile,implode("\n", $line));
echo ProgressBar::finish($wopt_clear);
echo ProgressBar::start($passed_file,"Running thumb batch (".stepString().")");
if ($wopt_parallel) {
echo ProgressBar::start($passed_file,"Running batch (".stepString().")");
if ($p['parallel']) {
passthru($bin_parallel." < ".$batchfile);
} else {
passthru("bash ".$batchfile);
@@ -585,183 +602,6 @@ if ($p['thumbs']) {
}
// Icons
//////////////////////////////////////////
if ($p['icons']) {
echo ProgressBar::start($passed_file,"Creating icon batch (".stepString().")",1);
$batchfile = $tmpdir."/_batch_icons.sh";
$helper = realpath("helper.php");
foreach ($fx as $count => $array) {
$fid = $array[0];
$pathname = $array[1];
$icmd = $bin_php." ".escapeshellarg($helper)." icons ".$fid." ".escapeshellarg($pathname)." ".escapeshellarg($bpath)." ".$mytime."; echo ".ProgressBar::next();
msg($icmd);
$iline[] = $icmd;
}
file_put_contents($batchfile,implode("\n", $iline));
echo ProgressBar::finish($wopt_clear);
echo ProgressBar::start($passed_file,"Running icon batch (".stepString().")");
if ($wopt_parallel) {
passthru($bin_parallel." < ".$batchfile);
} else {
passthru("bash ".$batchfile);
}
echo ProgressBar::finish($wopt_clear);
}
// External metadata
//////////////////////////////////////////
if ($p['meta']) {
echo ProgressBar::start(count($fx),"Collecting external metadata (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$shellpath = escapeshellarg($pathname);
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$found = 1;
if (!in_array($ext, $p['e_files']) && !in_array($ext, $p['m_files'])) {
echo ProgressBar::next("Not a media file: ".shortlabel($pathname));
continue;
}
if (in_array($ext, $p['e_files'])) {
$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch()[0];
if (!$check) {
$arrstring = shell_exec($bin_exiftool." -php ".$shellpath);
// $rawexif = eval("return ".`$bin_exiftool -php $shellpath`);
// do an addtl check below to prevent "PHP Parse error: syntax error, unexpected end of file, expecting ';'"
if (substr($arrstring,0,5) == "Array") {
$rawexif = eval("return ".$arrstring);
$stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":tags",serialize($rawexif[0]));
$stmt->execute();
$found = 0;
}
}
}
if (in_array($ext, $p['m_files'])) {
$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch()[0];
if (!$check) {
$stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)");
$stmt->BindValue(":fid",$fid);
//$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"))));
//$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"));
$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=JSON ".$shellpath." 2>&1"));
$stmt->execute();
$found = 0;
}
}
if ($found) {
echo ProgressBar::next("Metadata found: ".shortlabel($pathname));
} else {
echo ProgressBar::next("Collecting metadata: ".shortlabel($pathname));
}
}
echo ProgressBar::finish($wopt_clear);
}
// Hashes
//////////////////////////////////////////
if ($p['hash']) {
if ($p['hash_limit']) {
$message = "Generating hashes for files under ".$p['hash_limit']."GB";
} else {
$message = "Generating hashes for all files";
}
echo ProgressBar::start(count($fx),$message." (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$size = filesize($pathname);
$limit = $p['hash_limit']*1000000000;
$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0];
if ($check) {
echo ProgressBar::next("Hash already exists: ".shortlabel($pathname));
} elseif ($p['hash_limit'] && ($size > $limit)) {
echo ProgressBar::next("Too big to hash: ".shortlabel($pathname)." (".human_filesize($size).")");
} else {
echo ProgressBar::next("Generating hash: ".shortlabel($pathname));
$stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":hash",md5_file($pathname));
$stmt->execute();
}
}
echo ProgressBar::finish($wopt_clear);
}
// Contents
//////////////////////////////////////////
if ($p['contents']) {
// should be rewritten to check against filemtimes
echo ProgressBar::start(count($fx),"Gathering contents (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$cpath = $bpath."/contents/".substr($fid, 0, 2);
$cfile = $cpath."/".$fid.".zip";
if (in_array($ext, $p['c_files'])) {
if (!is_dir($cpath)) { mkdir($cpath); }
$max_size = $p['contents_limit'] * 1000;
if ($ext == ".DS_Store") {
// store all DS_Store files no matter how big
$my_size = 1;
} else {
$my_size = filesize($pathname);
}
if (!file_exists($cfile) && $my_size < $max_size) {
msg("Zipping ".$pathname);
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
$zip->addfile($pathname,basename($pathname));
$zip->close();
$stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($cfile, strlen($bpath)));
$stmt->execute();
}
}
echo ProgressBar::next(true);
}
echo ProgressBar::finish($wopt_clear);
}
// Pool Indices
//////////////////////////////////////////