p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.7.7

Browse Source
This commit is contained in:
profiteroles
2018-09-14 13:50:27 -07:00
parent a7716a08cc
commit 34de0a92c9
4 changed files with 218 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
Yuba.php
View File

@@ -3,7 +3,7 @@
// Yuba
// //
//////////////////////////////////////////
$version = "0.7.6";
$version = "0.7.7";
ini_set('memory_limit', '4096M');
date_default_timezone_set("America/Los_Angeles");
@@ -37,6 +37,7 @@ if (is_writable($zpath)) { echo "Warning: source is writeable\n"; }
$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle";
if (!is_dir($bpath)) { mkdir($bpath); }
if (!is_dir($bpath."/thumbs")) { mkdir($bpath."/thumbs"); }
if (!is_dir($bpath."/contents")) { mkdir($bpath."/contents"); }
// Treat these directories as files
$p['bundles'] = array( "app",
@@ -100,7 +101,7 @@ $disks = shell_exec("diskutil list 2>&1");
$df = shell_exec("df 2>&1");
$df_volume = shell_exec("df ".escapeshellarg($zpath)." | tail -n 1 | rev | cut -d' ' -f1 | rev");
$df_device = shell_exec("df ".escapeshellarg($zpath)." | tail -n 1 | cut -d' ' -f1");
$mdutil = shell_exec("mdutil -s ".escapeshellarg($df_volume));
$mdutil = shell_exec("mdutil -s ".$df_volume);
if (strpos($mdutil,"disabled")) {
echo "Warning: spotlight indexing is disabled\n";
$p['spotlight'] = false;
@@ -114,7 +115,7 @@ if (substr($zpath, 0, 9) != "/Volumes/") {
}
$diskutil = shell_exec("diskutil info ".$zbase." 2>&1");
$getstats = array( "Volume Name",
$getdetail = array( "Volume Name",
"Protocol",
"Volume UUID",
"Device Location",
@@ -122,24 +123,24 @@ $getstats = array( "Volume Name",
"Volume Available Space",
"Level Type"
);
foreach ($getstats as $stat) {
preg_match("/(".$stat.":)(.*)(\n)/",$diskutil,$matches);
foreach ($getdetail as $detail) {
preg_match("/(".$detail.":)(.*)(\n)/",$diskutil,$matches);
if (isset($matches[2])) {
if (substr($stat, -5, 5) == "Space") {
if (substr($detail, -5, 5) == "Space") {
$pieces = explode(" ", trim($matches[2]));
$summary = $pieces[0]." ".$pieces[1];
$stats[$stat] = $summary;
$details[$detail] = $summary;
} else {
$stats[$stat] = trim($matches[2]);
$details[$detail] = trim($matches[2]);
}
}
}
$dstats = serialize($stats);
$dstring = serialize($details);
if ($zpath == "/") {
$type = "Startup disk";
} elseif (strtolower($zpath) == strtolower("/Volumes/".$stats["Volume Name"])) {
if ($stats["Protocol"] == "Disk Image") {
} elseif (strtolower($zpath) == strtolower("/Volumes/".$details["Volume Name"])) {
if ($details["Protocol"] == "Disk Image") {
$type = "Disk image";
} else {
$type = "External disk";
@@ -164,13 +165,11 @@ echo "Building database...\n";
$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3");
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
/*
$dbo->query("PRAGMA page_size = 4096");
$dbo->query("PRAGMA cache_size = 10000");
$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
$dbo->query("PRAGMA synchronous = NORMAL");
$dbo->query("PRAGMA journal_mode = WAL");
*/
$dbo->exec("CREATE TABLE _skim (
version TEXT,
@@ -187,7 +186,7 @@ $dbo->exec("CREATE TABLE _skim (
nodescended INTEGER,
ignored INTEGER,
dupes INTEGER,
stats TEXT,
details TEXT,
qlmanage TEXT,
sysvers TEXT,
diskutil TEXT,
@@ -235,11 +234,10 @@ $dbo->exec("CREATE TABLE files (
thumb_filename TEXT,
thumb_width INTEGER,
thumb_height INTEGER,
has_contents INTEGER,
contents_filename TEXT
)");
$stmt = $dbo->prepare("INSERT INTO _skim VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)");
$stmt = $dbo->prepare("INSERT INTO _skim VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :details, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)");
$stmt->BindValue(":version",$version);
$stmt->BindValue(":opts",serialize($p));
$stmt->BindValue(":host",$host);
@@ -247,7 +245,7 @@ $stmt->BindValue(":uid",posix_getuid());
$stmt->BindValue(":zpath",$zpath);
$stmt->BindValue(":bpath",$bpath);
$stmt->BindValue(":type",$type);
$stmt->BindValue(":stats",$dstats);
$stmt->BindValue(":details",$dstring);
$stmt->BindValue(":qlmanage",$qlmanage);
$stmt->BindValue(":sysvers",$sysvers);
$stmt->BindValue(":diskutil",$diskutil);
@@ -329,7 +327,15 @@ echo "Total files: ".$passed_total."\n";
//////////////////////////////////////////
$dbp = new PDO("sqlite:".$bpath."/pool.sqlite3");
$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbp->query("PRAGMA page_size = 4096");
$dbp->query("PRAGMA cache_size = 10000");
$dbp->query("PRAGMA locking_mode = EXCLUSIVE");
$dbp->query("PRAGMA synchronous = NORMAL");
$dbp->query("PRAGMA journal_mode = WAL");
$dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)");
@@ -372,20 +378,18 @@ foreach ($files as $splFileInfo) {
$family[$key]['fid'] = $fid;
}
// capture stat before values change
// Cache stat values in stx array
/*
if ($splFileInfo->getType() != "link") {
$stx[$i] = array( $splFileInfo->getATime(),
$splFileInfo->getMTime(),
$splFileInfo->getCTime() );
}
*/
if ($p['bypass_stat']) {
$sty[$i] = "bypass";
} else {
$sty[$i] = chop(shell_exec("stat -x ".$shellpath." 2>&1"));
}
$sty[$i] = statToArray(shell_exec("stat -s ".$shellpath." 2>&1"));
// Check file can be read
@@ -474,18 +478,6 @@ $dbo->exec($stmt);
$wopt_currstep++;
// Contents
//////////////////////////////////////////
if ($p['contents']) {
echo "DO CONTENTS HERE\n";
// make a dir in the bundle called contents (similar to thumbs)
// match files smaller than x and with file extension of txt etc
// copy files to hash dirs in bundle (like thumbs dir)
}
// Thumbnails
//////////////////////////////////////////
@@ -564,6 +556,42 @@ if ($p['thumbs']) {
}
// Contents
//////////////////////////////////////////
if ($p['contents']) {
// should be rewritten to check against filemtimes
echo ProgressBar::start(count($fx),"Gathering contents (".stepString().")");
foreach ($fx as $array) {
$fid = $array[0];
$pathname = $array[1];
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
$cpath = $bpath."/contents/".substr($fid, 0, 2);
$cfile = $cpath."/".$fid.".zip";
if (in_array($ext, $c_files)) {
if (!is_dir($cpath)) { mkdir($cpath); }
if (!file_exists($cfile) && filesize($pathname) < 25000) {
$stmt = $dbp->prepare("INSERT INTO contents VALUES (:fid, :created, :relative_path)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$stmt->BindValue(":relative_path",substr($cfile, strlen($bpath)));
$stmt->execute();
$zip = new ZipArchive();
$zip->open($cfile, ZipArchive::CREATE);
$zip->addfile($pathname,basename($pathname));
$zip->close();
}
}
echo ProgressBar::next(true);
}
echo ProgressBar::finish();
}
// External metadata
//////////////////////////////////////////
@@ -754,54 +782,6 @@ if ($p['spotlight']) {
}
// Milk
//////////////////////////////////////////
// Aggregate values
$sb['a'] = array( "t*Title" => array("e_Title","k_Title","m_Title"),
"t*Dimensions" => array("k_PixelWidth.k_PixelHeight","e_PixelWidth.e_PixelHeight","m_PixelWidth.m_PixelHeight"),
"i*Seconds" => array("k_DurationSeconds","e_Duration","m_Duration"),
"d*DateTime" => array("e_DateTimeOriginal","m_EncodedDate","e_CreateDate","e_MediaCreateDate","k_ContentCreationDate"),
"t*Origin" => array("e_CameraModelName","e_Producer","e_CreatorTool","e_WriterName","e_Software","e_Encoder","m_WritingApplication"),
"t*GPS" => array("e_GPSPosition","k_Latitude.k_Longitude"),
"t*Author" => array("e_Author","e_Artist","e_Creator","e_By-line")
);
// Exiftool values
$sb['e'] = array( "i*Orientation",
"t*Compression",
"t*ProfileDescription",
"i*BitDepth",
"t*LensType",
"t*FocalLength",
"t*Aperture",
"t*LightSource",
"t*WhiteBalance" );
// Mediainfo values
$sb['m'] = array( "t*VideoFormat",
"t*AudioFormat",
"i*Tracks",
"t*Profile",
"t*Bitrate" );
// Build DB
unset($cbuild, $ibuild);
foreach (array_merge(array_keys($sb['a']),$sb['e'],$sb['m']) as $name) {
$parts = explode("*",$name);
$ibuild[] = ":".$parts[1];
if ($parts[0] == "t") {
$cbuild[] = $parts[1]." TEXT";
} else {
$cbuild[] = $parts[1]." INTEGER";
}
}
$dbo->exec("CREATE TABLE milk (".implode(",",$cbuild).")");
$stmt = $dbo->prepare("INSERT INTO milk VALUES (".implode(",",$ibuild).")");
$stmt->execute();
// Files
//////////////////////////////////////////
@@ -813,7 +793,7 @@ foreach ($files as $splFileInfo) {
// DB
$stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Size, :Inode, :Perms, :Owner, :ATime, :CTime, :MTime, :LinkTarget, :RealPath, :stat, :items, :newest, :gfi_type, :gfi_attr, :gfi_created, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :has_contents, :contents_filename)");
$stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Size, :Inode, :Perms, :Owner, :ATime, :MTime, :CTime, :LinkTarget, :RealPath, :stat, :items, :newest, :gfi_type, :gfi_attr, :gfi_created, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :contents_filename)");
// Identify dir, file, link or bundle dir
@@ -841,7 +821,7 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":Extension",$extension);
//stat
$stmt->BindValue(":stat",$sty[$j]);
$stmt->BindValue(":stat",serialize($sty[$j]));
if ($type == "link") {
@@ -854,9 +834,9 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":Perms",$splFileInfo->getPerms());
$stmt->BindValue(":Owner",$splFileInfo->getOwner().":".$splFileInfo->getGroup());
$stmt->BindValue(":ATime",$stx[$j][0]);
$stmt->BindValue(":MTime",$stx[$j][1]);
$stmt->BindValue(":CTime",$stx[$j][2]);
$stmt->BindValue(":ATime",$splFileInfo->getATime());
$stmt->BindValue(":MTime",$splFileInfo->getMTime());
$stmt->BindValue(":CTime",$splFileInfo->getCTime());
}
@@ -928,16 +908,17 @@ foreach ($files as $splFileInfo) {
unset($fetch_exif, $fetch_media, $fetch_hash, $fetch_thumb, $yes_exif, $yes_media, $yes_hash);
$fetch_exif = @unserialize($dbp->query("SELECT tags FROM exiftool WHERE fid='".$fid."'")->fetch()[0]);
is_array($fetch_exif) ? $yes_exif = 1 : $yes_exif = 0;
$yes_exif = $dbp->query("SELECT rowid FROM exiftool WHERE fid='".$fid."'")->fetch()[0];
$stmt->BindValue(":has_exif",$yes_exif);
$fetch_media = @unserialize($dbp->query("SELECT info FROM mediainfo WHERE fid='".$fid."'")->fetch()[0]);
is_array($fetch_media) ? $yes_media = 1 : $yes_media = 0;
$yes_media = $dbp->query("SELECT rowid FROM mediainfo WHERE fid='".$fid."'")->fetch()[0];
$stmt->BindValue(":has_mediainfo",$yes_media);
$yes_hash = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0];
$yes_hash = $dbp->query("SELECT rowid FROM md5 WHERE fid='".$fid."'")->fetch()[0];
$stmt->BindValue(":has_hash",$yes_hash);
$yes_contents = $dbp->query("SELECT relative_path FROM contents WHERE fid='".$fid."'")->fetch()[0];
$stmt->BindValue(":contents_filename",$yes_contents);
$fetch_thumb = $dbp->query("SELECT * FROM thumbs WHERE fid='".$fid."'")->fetch();
if (@$fetch_thumb['relative_path']) {
@@ -950,63 +931,35 @@ foreach ($files as $splFileInfo) {
}
// ------------------------------------------------ //
// Milk
// ------------------------------------------------ //
// Write to DB
$stmt->execute();
// Set fileatime back to original value
//if ($type != "link" && is_writable($pathname) && $p['fixatimes']) {
// exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1");
// }
// Double check stat for file against pre-run value
if ($p['verify_stat'] && $type != "link") {
$pre_access = $pre_modify = $pre_change = null;
$post_access = $post_modify = $post_change = null;
foreach (explode("\n", $sty[$j]) as $line) {
$check = substr($line, 0, 6);
if ($check == "Access") { $pre_access = $line; }
if ($check == "Modify") { $pre_modify = $line; }
if ($check == "Change") { $pre_change = $line; }
}
foreach (explode("\n", chop(shell_exec("stat -x ".$shellpath." 2>&1"))) as $line) {
$check = substr($line, 0, 6);
if ($check == "Access") { $post_access = $line; }
if ($check == "Modify") { $post_modify = $line; }
if ($check == "Change") { $post_change = $line; }
}
$restat = statToArray(shell_exec("stat -s ".$shellpath." 2>&1"));
$message = array();
if ($pre_access != $post_access) {
$message[] = "ATIME";
if ($sty[$j]['st_atime'] != $restat['st_atime']) {
if ($p['fixatimes'] && $type != "link" && is_writable($pathname)) {
exec("touch -at `date -r ".$sty[$j]['st_atime']." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1");
$message[] = "atime (fix)";
} else {
$message[] = "atime";
}
}
if ($pre_modify != $post_modify) {
$message[] = "MTIME";
if ($sty[$j]['st_mtime'] != $restat['st_mtime']) {
$message[] = "mtime";
}
if ($pre_change != $post_change) {
$message[] = "CTIME";
if ($sty[$j]['st_ctime'] != $restat['st_ctime']) {
$message[] = "ctime";
}
if (count($message)) { echo "\nChanged: ".implode(", ", $message)."\n"; }
if (count($message)) { echo "\nFILE = ".$filename."; CHANGE = ".implode(", ", $message)."\n"; }
}
@@ -1017,6 +970,60 @@ foreach ($files as $splFileInfo) {
echo ProgressBar::finish();
// Milk
//////////////////////////////////////////
// Aggregate values
$sb['a'] = array( "t*Title" => array("e_Title","k_Title","m_Title"),
"t*Dimensions" => array("k_PixelWidth.k_PixelHeight","e_PixelWidth.e_PixelHeight","m_PixelWidth.m_PixelHeight"),
"i*Seconds" => array("k_DurationSeconds","e_Duration","m_Duration"),
"d*DateTime" => array("e_DateTimeOriginal","m_EncodedDate","e_CreateDate","e_MediaCreateDate","k_ContentCreationDate"),
"t*Origin" => array("e_CameraModelName","e_Producer","e_CreatorTool","e_WriterName","e_Software","e_Encoder","m_WritingApplication"),
"t*GPS" => array("e_GPSPosition","k_Latitude.k_Longitude"),
"t*Author" => array("e_Author","e_Artist","e_Creator","e_By-line")
);
// Exiftool values
$sb['e'] = array( "i*Orientation",
"t*Compression",
"t*ProfileDescription",
"i*BitDepth",
"t*LensType",
"t*FocalLength",
"t*Aperture",
"t*LightSource",
"t*WhiteBalance" );
// Mediainfo values
$sb['m'] = array( "t*VideoFormat",
"t*AudioFormat",
"i*Tracks",
"t*Profile",
"t*Bitrate" );
// Build DB
unset($cbuild, $ibuild);
foreach (array_merge(array_keys($sb['a']),$sb['e'],$sb['m']) as $name) {
$parts = explode("*",$name);
$ibuild[] = ":".$parts[1];
if ($parts[0] == "t") {
$cbuild[] = $parts[1]." TEXT";
} else {
$cbuild[] = $parts[1]." INTEGER";
}
}
$dbo->exec("CREATE TABLE milk (".implode(",",$cbuild).")");
$stmt = $dbo->prepare("INSERT INTO milk VALUES (".implode(",",$ibuild).")");
$stmt->execute();
//$q = $dbo->query("SELECT rowid, * FROM files");
//while ($row = $q->fetch_assoc()) {
// print_r($row);
// }
//$q->free();
// Cleanup
//////////////////////////////////////////
@@ -1033,9 +1040,9 @@ if ($p['rsync_dest']) {
echo "\nrsync...\n";
$command = "rsync -avv -e ssh ".$bpath." ".$p['rsync_dest'];
$count = trim(shell_exec("find ".escapeshellarg($bpath)." | wc -l"));
echo ProgressBar::start($count,"rsync");
echo ProgressBar::start($count,$p['rsync_dest']);
$pipe = popen($command, "r");
while(fgets($pipe, 2048)) { echo ProgressBar::next(); }
while(fgets($pipe, 2048)) { echo ProgressBar::next(true); }
pclose($pipe);
echo ProgressBar::finish();
}