p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.7.9

Browse Source
This commit is contained in:
profiteroles
2018-09-16 15:22:15 -07:00
parent 34de0a92c9
commit 01d7489c05
3 changed files with 502 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

267
Yuba.php
View File

@@ -3,7 +3,7 @@
// Yuba
// //
//////////////////////////////////////////
$version = "0.7.7";
$version = "0.7.9";
ini_set('memory_limit', '4096M');
date_default_timezone_set("America/Los_Angeles");
@@ -14,8 +14,8 @@ date_default_timezone_set("America/Los_Angeles");
require("functions.php");
require("filetypes.php");
$wopt_noprofile = 1;
$wopt_steps = 9;
$wopt_noprofile = 0;
$wopt_steps = 10;
$wopt_currstep = 1;
$p = unserialize(file_get_contents("prefs.php"));
@@ -32,7 +32,7 @@ if (!is_dir($zpath) | !is_dir($bdest)) { echo "Filepath error"; die; }
// Check for bundle
if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); }
if (is_writable($zpath)) { echo "Warning: source is writeable\n"; }
if (is_writable($zpath)) { echo "Warning: source is writeable"; }
$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle";
if (!is_dir($bpath)) { mkdir($bpath); }
@@ -79,6 +79,7 @@ $bin_qlthumb = __DIR__."/bin/ql-thumbnail";
// Logfile
$messages_log_file = $bpath."/".$stamp."_messages.log";
$error_log_file = $bpath."/".$stamp."_error.log";
error_reporting(E_ALL);
ini_set("display_errors", TRUE);
@@ -89,21 +90,21 @@ ini_set("error_log", $error_log_file);
//////////////////////////////////////////
$banner = "Yuba: ".$zpath." -> ".$bpath;
echo $banner."\n".str_repeat("-", strlen($banner))."\n";
echo msg($banner."\n".str_repeat("-", strlen($banner)));
// Disk info
//////////////////////////////////////////
echo "Gathering system info...\n";
echo msg("Gathering system info...");
$host = gethostname();
$disks = shell_exec("diskutil list 2>&1");
$disks = shell_exec("diskutil list -plist 2>&1");
$df = shell_exec("df 2>&1");
$df_volume = shell_exec("df ".escapeshellarg($zpath)." | tail -n 1 | rev | cut -d' ' -f1 | rev");
$df_device = shell_exec("df ".escapeshellarg($zpath)." | tail -n 1 | cut -d' ' -f1");
$mdutil = shell_exec("mdutil -s ".$df_volume);
if (strpos($mdutil,"disabled")) {
echo "Warning: spotlight indexing is disabled\n";
echo msg("Warning: spotlight indexing is disabled");
$p['spotlight'] = false;
}
@@ -114,6 +115,13 @@ if (substr($zpath, 0, 9) != "/Volumes/") {
$zbase = "/Volumes/".$zparts[2];
}
$hdiutil = shell_exec("hdiutil info -plist 2>&1");
/////////////////////////////////////////////////////////////////////////////////////////////
// rewrite below to use diskutil info -plist
//
//
//
$diskutil = shell_exec("diskutil info ".$zbase." 2>&1");
$getdetail = array( "Volume Name",
"Protocol",
@@ -148,6 +156,11 @@ if ($zpath == "/") {
} else {
$type = "Folder";
}
//
//
//
/////////////////////////////////////////////////////////////////////////////////////////////
if ($wopt_noprofile) {
$profile = "disabled";
@@ -160,10 +173,11 @@ $sysvers = shell_exec("sw_vers 2>&1");
// Database
//////////////////////////////////////////
echo "Building database...\n";
echo msg("Building database...");
$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3");
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
$dbo->query("PRAGMA page_size = 4096");
$dbo->query("PRAGMA cache_size = 10000");
@@ -190,6 +204,7 @@ $dbo->exec("CREATE TABLE _skim (
qlmanage TEXT,
sysvers TEXT,
diskutil TEXT,
hdiutil TEXT,
disks TEXT,
df TEXT,
df_device TEXT,
@@ -237,7 +252,7 @@ $dbo->exec("CREATE TABLE files (
contents_filename TEXT
)");
$stmt = $dbo->prepare("INSERT INTO _skim VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :details, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)");
$stmt = $dbo->prepare("INSERT INTO _skim VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :details, :qlmanage, :sysvers, :diskutil, :hdiutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)");
$stmt->BindValue(":version",$version);
$stmt->BindValue(":opts",serialize($p));
$stmt->BindValue(":host",$host);
@@ -249,6 +264,7 @@ $stmt->BindValue(":details",$dstring);
$stmt->BindValue(":qlmanage",$qlmanage);
$stmt->BindValue(":sysvers",$sysvers);
$stmt->BindValue(":diskutil",$diskutil);
$stmt->BindValue(":hdiutil",$hdiutil);
$stmt->BindValue(":disks",$disks);
$stmt->BindValue(":df",$df);
$stmt->BindValue(":df_device",$df_device);
@@ -311,17 +327,17 @@ $files = new RecursiveIteratorIterator(
// Tally
//////////////////////////////////////////
echo "Counting files...\n";
echo msg("Counting files...");
foreach ($files as $null) { }
$first_run = 0;
if (!$passed_total) {
echo "Nothing was found, exiting";
echo msg("Nothing was found, exiting");
die;
}
echo "Total files: ".$passed_total."\n";
echo msg("Total files: ".$passed_total."");
// Pool DB
//////////////////////////////////////////
@@ -363,7 +379,7 @@ foreach ($files as $splFileInfo) {
$pkey = md5($path);
if (array_key_exists($key, $family)) {
echo "Duplicate key on ".$pathname."\n"; die;
echo msg("Duplicate key on ".$pathname.""); die;
}
$family[$key] = array();
@@ -389,6 +405,7 @@ foreach ($files as $splFileInfo) {
}
*/
//$sty[$i] = "bypass";
$sty[$i] = statToArray(shell_exec("stat -s ".$shellpath." 2>&1"));
// Check file can be read
@@ -412,12 +429,12 @@ echo ProgressBar::finish();
// Thow permissions error
if (count($noread)) {
echo "Current user (".posix_getuid().") does not have read access to the following files:\n";
echo msg("Current user (".posix_getuid().") does not have read access to the following files:");
foreach ($noread as $file) {
echo $file."\n";
}
if ($p['readability']) {
echo "Exiting...";
echo msg("Exiting...");
die;
}
}
@@ -574,15 +591,20 @@ if ($p['contents']) {
if (in_array($ext, $c_files)) {
if (!is_dir($cpath)) { mkdir($cpath); }
if (!file_exists($cfile) && filesize($pathname) < 25000) {
msg("Zipping ".$pathname);
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
$zip->addfile($pathname,basename($pathname));
$zip->close();
$stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($cfile, strlen($bpath)));
$stmt->execute();
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
$zip->addfile($pathname,basename($pathname));
$zip->close();
}
}
echo ProgressBar::next(true);
@@ -689,6 +711,17 @@ if ($p['hash']) {
}
// Pool Indices
//////////////////////////////////////////
// We are done with the Pool DB, make sure there are indices
$dbp->exec("CREATE INDEX IF NOT EXISTS contents_index ON contents (fid)");
$dbp->exec("CREATE INDEX IF NOT EXISTS exiftool_index ON exiftool (fid)");
$dbp->exec("CREATE INDEX IF NOT EXISTS md5_index ON md5 (fid)");
$dbp->exec("CREATE INDEX IF NOT EXISTS mediainfo_index ON mediainfo (fid)");
$dbp->exec("CREATE INDEX IF NOT EXISTS thumbs_index ON thumbs (fid)");
// Spotlight
//////////////////////////////////////////
@@ -741,13 +774,18 @@ if ($p['spotlight']) {
foreach ($files as $splFileInfo) {
$path = $splFileInfo->getPathname();
msg($path);
$pid = md5($path);
$shellpath = escapeshellarg($path);
$mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1");
if (substr_count(@$mdls,"\n") < 2) { continue; }
$parser = new plistParser();
$spotlight = $parser->parseString($mdls);
if (substr_count(@$mdls,"\n") > 1) {
$parser = new plistParser();
$spotlight = $parser->parseString(utf8_for_xml($mdls));
} else {
$spotlight = array();
}
$stmt = $dbo->prepare("INSERT INTO mdls VALUES (".implode(",",$ibuild).")");
@@ -868,7 +906,10 @@ foreach ($files as $splFileInfo) {
// Items
if ($type == "dir" || $type == "bundle" ) {
$items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\..*' \) | wc -l 2>&1"))-1;
// below commented out because it was causing -1 on dirs beginning with a dot
//$items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\..*' \) | wc -l 2>&1"))-1;
// below should be rewritten to use $wopt_ignore files
$items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\.DS_Store' \) | wc -l 2>&1"))-1;
$stmt->BindValue(":items",@$items);
}
@@ -959,11 +1000,11 @@ foreach ($files as $splFileInfo) {
$message[] = "ctime";
}
if (count($message)) { echo "\nFILE = ".$filename."; CHANGE = ".implode(", ", $message)."\n"; }
if (count($message)) { echo msg("FILE = ".$filename."; CHANGE = ".implode(", ", $message).""); }
}
echo ProgressBar::next($filename);
echo ProgressBar::next($pathname);
$j++;
}
@@ -973,61 +1014,147 @@ echo ProgressBar::finish();
// Milk
//////////////////////////////////////////
// Aggregate values
$sb['a'] = array( "t*Title" => array("e_Title","k_Title","m_Title"),
"t*Dimensions" => array("k_PixelWidth.k_PixelHeight","e_PixelWidth.e_PixelHeight","m_PixelWidth.m_PixelHeight"),
"i*Seconds" => array("k_DurationSeconds","e_Duration","m_Duration"),
"d*DateTime" => array("e_DateTimeOriginal","m_EncodedDate","e_CreateDate","e_MediaCreateDate","k_ContentCreationDate"),
"t*Origin" => array("e_CameraModelName","e_Producer","e_CreatorTool","e_WriterName","e_Software","e_Encoder","m_WritingApplication"),
"t*GPS" => array("e_GPSPosition","k_Latitude.k_Longitude"),
"t*Author" => array("e_Author","e_Artist","e_Creator","e_By-line")
);
$milk['t*Title'] = ["e^Title","k^Title","m^Track_name"];
$milk['t*Format'] = ["m^Format","e^Compression","e^MIMEType"];
$milk['t*Dimensions'] = ["k^PixelWidth.k^PixelHeight","e^PixelWidth.e^PixelHeight","m^SkimDims"];
$milk['s*Seconds'] = ["k^DurationSeconds","e^Duration","m^Duration"];
$milk['d*DateTime'] = ["e^DateTimeOriginal","m^EncodedDate","e^CreateDate","e^MediaCreateDate","k^ContentCreationDate"];
$milk['t*Origin'] = ["e^CameraModelName","e^Producer","e^CreatorTool","e^WriterName","e^Software","e^Encoder","k^Creator"];
$milk['t*GPS'] = ["k^Latitude.k^Longitude","e^GPSPosition"];
$milk['t*Author'] = ["e^Author","e^Artist","e^Creator","e^By-line"];
// Exiftool values
$sb['e'] = array( "i*Orientation",
"t*Compression",
"t*ProfileDescription",
"i*BitDepth",
"t*LensType",
"t*FocalLength",
"t*Aperture",
"t*LightSource",
"t*WhiteBalance" );
$milk['i*Tracks'] = ["m^SkimTrackCount"];
$milk['t*Writer'] = ["m^Writing_application.m^Writing_library"];
$milk['t*Bitrate'] = ["m^Overall_bit_rate"];
// Mediainfo values
$sb['m'] = array( "t*VideoFormat",
"t*AudioFormat",
"i*Tracks",
"t*Profile",
"t*Bitrate" );
$milk['i*Orientation'] = ["e^Orientation"];
$milk['t*Profile'] = ["e^Profile"];
$milk['i*BitDepth'] = ["e^BitDepth"];
$milk['t*LensType'] = ["e^LensType"];
$milk['t*FocalLength'] = ["e^FocalLength"];
$milk['t*Aperture'] = ["e^Aperture"];
$milk['t*LightSource'] = ["e^LightSource"];
$milk['t*WhiteBalance'] = ["e^WhiteBalance"];
$delimiter = ",";
// Build DB
unset($cbuild, $ibuild);
foreach (array_merge(array_keys($sb['a']),$sb['e'],$sb['m']) as $name) {
$parts = explode("*",$name);
$ibuild[] = ":".$parts[1];
if ($parts[0] == "t") {
$cbuild[] = $parts[1]." TEXT";
} else {
$cbuild[] = $parts[1]." INTEGER";
$cbuild = $ibuild = array();
foreach (array_keys($milk) as $name) {
list($kind,$item) = explode("*",$name);
switch ($kind) {
case "t":
case "d":
case "s":
$cbuild[] = $item." TEXT";
break;
case "i":
$cbuild[] = $item." INTEGER";
break;
}
$ibuild[] = ":".$item;
}
$dbo->exec("CREATE TABLE milk (".implode(",",$cbuild).")");
$stmt = $dbo->prepare("INSERT INTO milk VALUES (".implode(",",$ibuild).")");
$stmt->execute();
$countrows = @reset($dbo->query("SELECT max(rowid) FROM files")->fetch());
echo msg("Milking ".$countrows." rows");
echo ProgressBar::start($countrows, "Milk");
//$q = $dbo->query("SELECT rowid, * FROM files");
//while ($row = $q->fetch_assoc()) {
// print_r($row);
// }
//$q->free();
$loop = $dbo->query("SELECT rowid, * FROM files");
while ($row_a = $loop->fetch()) {
$stmt = $dbo->prepare("INSERT INTO milk VALUES (".implode(",",$ibuild).")");
$row_b = @$dbo->query("SELECT * FROM mdls WHERE (rowid='".$row_a['rowid']."')")->fetch();
if (count(@$row_b) > 1) {
$m['k'] = $row_b;
} else {
$m['k'] = null;
}
if (isset($row_a['has_exif'])) {
$row_c = $dbp->query("SELECT * FROM exiftool WHERE (rowid='".$row_a['has_exif']."')")->fetch();
$m['e'] = unserialize($row_c['tags']);
} else {
$m['e'] = null;
}
if (isset($row_a['has_mediainfo'])) {
$row_d = $dbp->query("SELECT * FROM mediainfo WHERE (rowid='".$row_a['has_mediainfo']."')")->fetch();
$decoded = @json_decode(json_encode(simplexml_load_string($row_d['info'])),true);
$m['m'] = $decoded['File']['track'][0];
//custom values
$m['m']['SkimTrackCount'] = @count($decoded['File']['track']);
@foreach ($decoded['File']['track'] as $track) {
if (!@$m['m']['SkimDims'] && @$track['Width'] && @$track['Height']) {
$m['m']['SkimDims'] = sanitize($track['Width'],"i").$delimiter.sanitize($track['Height'],"i");
}
}
// do seconds fix here
} else {
$m['m'] = null;
}
// M*I*L*K baby
foreach ($milk as $value => $weighted) {
list($type,$name) = explode("*",$value);
$found = 0;
foreach ($weighted as $dindex) {
// concatenante 2 values
if (!$found && strpos($dindex, ".")) {
$parts = explode(".",$dindex);
$out = array();
foreach ($parts as $part) {
list($kind,$item) = explode("^",$part);
if (@$m[$kind][$item]) {
$out[] = sanitize($m[$kind][$item],$type);
}
}
if (count($out)) {
$stmt->BindValue(":".$name,implode($delimiter,$out));
$found = 1;
}
} elseif (!$found) {
// find a single value
list($kind,$item) = explode("^",$dindex);
if (@$m[$kind][$item]) {
$stmt->BindValue(":".$name,sanitize($m[$kind][$item],$type));
$found = 1;
}
}
}
}
echo ProgressBar::next(true);
$stmt->execute();
}
echo ProgressBar::finish();
// Cleanup
//////////////////////////////////////////
echo "\n";
echo msg("");
if (file_exists($error_log_file)) { echo file_get_contents($error_log_file); }
@@ -1037,7 +1164,7 @@ $dbo->exec("UPDATE _skim SET status='completed_in_".$seconds."'");
// rsync
if ($p['rsync_dest']) {
echo "\nrsync...\n";
echo msg("rsync...");
$command = "rsync -avv -e ssh ".$bpath." ".$p['rsync_dest'];
$count = trim(shell_exec("find ".escapeshellarg($bpath)." | wc -l"));
echo ProgressBar::start($count,$p['rsync_dest']);
@@ -1047,7 +1174,7 @@ if ($p['rsync_dest']) {
echo ProgressBar::finish();
}
echo "Finished in ".$seconds." seconds\n\n";
echo msg("Finished in ".$seconds." seconds");
unset($dbo, $dbp, $files, $family, $fx);