From d2ed189d2b6db866d2c1e687bd1a6fa6910eed7c Mon Sep 17 00:00:00 2001 From: profiteroles Date: Sun, 2 Sep 2018 11:21:24 -0700 Subject: [PATCH] 0.7.2.5 --- Yuba.php | 301 +++++++++++++++++++-------------------------- functions.php | 2 +- web/rtc.php | 60 ++++++--- web/rtc_legacy.php | 1 + 4 files changed, 172 insertions(+), 192 deletions(-) diff --git a/Yuba.php b/Yuba.php index 1df2abb..efc5eb4 100755 --- a/Yuba.php +++ b/Yuba.php @@ -3,7 +3,7 @@ // Yuba // // ////////////////////////////////////////// -$version = "0.7.2"; +$version = "0.7.2.5"; ini_set('memory_limit', '4096M'); date_default_timezone_set("America/Los_Angeles"); @@ -22,6 +22,8 @@ $p = unserialize(file_get_contents("prefs.php")); // Path & application variables ////////////////////////////////////////// +$stamp = date("Y-m-d_H-i-s", time()); + if (!isset($argv[1])) { echo "Input error"; die; } $zpath = realpath(@$argv[1]); if (@$argv[2]) { $bdest = realpath($argv[2]); } else { $bdest = realpath($p['bdest']); } @@ -75,10 +77,11 @@ $bin_qlthumb = __DIR__."/bin/ql-thumbnail"; // Logfile +$error_log_file = $bpath."/".$stamp."_error.log"; error_reporting(E_ALL); ini_set("display_errors", TRUE); ini_set("log_errors", TRUE); -ini_set("error_log", $bpath."/php.log"); +ini_set("error_log", $error_log_file); // Banner ////////////////////////////////////////// @@ -99,6 +102,7 @@ $df_device = shell_exec("df ".$zpath." | tail -n 1 | cut -d' ' -f1"); $mdutil = shell_exec("mdutil -s ".$df_volume); if (strpos($mdutil,"disabled")) { echo "Warning: spotlight indexing is disabled\n"; + $p['spotlight'] = false; } if (substr($zpath, 0, 9) != "/Volumes/") { @@ -156,8 +160,6 @@ $sysvers = shell_exec("sw_vers 2>&1"); echo "Building database...\n"; -$stamp = date("Y-m-d_H-i-s", time()); - $dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3"); $dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); @@ -169,7 +171,7 @@ $dbo->query("PRAGMA synchronous = NORMAL"); $dbo->query("PRAGMA journal_mode = WAL"); */ -$dbo->exec("CREATE TABLE _walkwalk ( +$dbo->exec("CREATE TABLE _skim ( version TEXT, opts TEXT, host TEXT, @@ -215,8 +217,8 @@ $dbo->exec("CREATE TABLE files ( Perms INTEGER, Owner TEXT, ATime INTEGER, - CTime INTEGER, MTime INTEGER, + CTime INTEGER, LinkTarget TEXT, RealPath TEXT, stat TEXT, @@ -234,23 +236,6 @@ $dbo->exec("CREATE TABLE files ( Origin TEXT, GPS TEXT, Author TEXT, - spotlight TEXT, - kMDItemDateAdded INTEGER, - kMDItemLastUsedDate INTEGER, - kMDItemUseCount INTEGER, - kMDItemContentModificationDate INTEGER, - kMDItemContentType TEXT, - kMDItemCreator TEXT, - kMDItemFSCreatorCode TEXT, - kMDItemKind TEXT, - kMDItemFSTypeCode TEXT, - kMDItemUserTags TEXT, - kMDItemFSInvisible INTEGER, - kMDItemNumberOfPages INTEGER, - kMDItemPageHeight INTEGER, - kMDItemPageWidth INTEGER, - kMDItemWhereFroms TEXT, - kMDItemEncodingApplications TEXT, has_exif INTEGER, has_mediainfo INTEGER, has_hash INTEGER, @@ -269,7 +254,7 @@ $dbo->exec("CREATE TABLE files ( Bitrate INTEGER )"); -$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)"); +$stmt = $dbo->prepare("INSERT INTO _skim VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)"); $stmt->BindValue(":version",$version); $stmt->BindValue(":opts",serialize($p)); $stmt->BindValue(":host",$host); @@ -346,6 +331,11 @@ $files = new RecursiveIteratorIterator( foreach ($files as $null) { } $first_run = 0; +if (!$passed_total) { + echo "Nothing was found, exiting"; + die; + } + // Permissions & Stat ////////////////////////////////////////// @@ -488,7 +478,7 @@ echo ProgressBar::finish(); unset($dx, $dxo, $dupes); // stats -$stmt = "UPDATE _walkwalk SET "; +$stmt = "UPDATE _skim SET "; $stmt .= "passed_file=".$passed_file.", "; $stmt .= "passed_dir=".$passed_dir.", "; $stmt .= "passed_link=".$passed_link.", "; @@ -504,6 +494,9 @@ $dbo->exec($stmt); if ($p['contents']) { echo "DO CONTENTS HERE\n"; + // make a dir in the bundle called contents (similar to db) + // match files smaller than x and with file extension of txt etc + // copy files to hash dirs in bundle (like db dir) } @@ -669,6 +662,98 @@ if ($p['hash']) { } +// Spotlight +////////////////////////////////////////// + +$mb['i'] = array( "PixelWidth", + "PixelHeight", + "Latitude", + "Longitude", + "DurationSeconds", + "UseCount", + "FSInvisible", + "NumberOfPages", + "PageHeight", + "PageWidth" ); + +$mb['t'] = array( "Title", + "ContentType", + "Creator", + "FSCreatorCode", + "Kind", + "FSTypeCode" ); + +$mb['a'] = array( "UserTags", + "WhereFroms", + "EncodingApplications" ); + +$mb['d'] = array( "DateAdded", + "LastUsedDate", + "ContentModificationDate", + "ContentCreationDate" ); + +$ibuild[] = ":pid, :spotlight"; +$cbuild[] = "pid TEXT, spotlight TEXT"; +foreach (array_merge($mb['i'],$mb['d']) as $item) { + $cbuild[] = $item." INTEGER"; + $ibuild[] = ":".$item; + } +foreach (array_merge($mb['t'],$mb['a']) as $item) { + $cbuild[] = $item." TEXT"; + $ibuild[] = ":".$item; + } + +$p['spotlight'] = 1; + +if ($p['spotlight']) { + + echo ProgressBar::start($passed_total,"Spotlight"); + + $dbo->exec("CREATE TABLE mdls (".implode(",",$cbuild).")"); + + foreach ($files as $splFileInfo) { + + $pid = md5($splFileInfo->getPathname()); + $shellpath = escapeshellarg($splFileInfo->getPathname()); + $mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1"); + if (substr_count(@$mdls,"\n") < 2) { continue; } + + $parser = new plistParser(); + $spotlight = $parser->parseString($mdls); + + $stmt = $dbo->prepare("INSERT INTO mdls VALUES (".implode(",",$ibuild).")"); + + foreach ($mb as $key => $list) { + foreach ($list as $item) { + + if (@$spotlight["kMDItem".$item]) { + $stmt->BindValue(":pid",$pid); + $stmt->BindValue(":spotlight",$mdls); + switch($key) { + case "i": + case "t": + $stmt->BindValue(":".$item,$spotlight["kMDItem".$item]); + break; + case "a": + $stmt->BindValue(":".$item,serialize($spotlight["kMDItem".$item])); + break; + case "d": + $stmt->BindValue(":".$item,strtotime($spotlight["kMDItem".$item])); + break; + } + } + } + } + + $stmt->execute(); + echo ProgressBar::next(); + + } + + echo ProgressBar::finish(); + + } + // Files ////////////////////////////////////////// @@ -682,7 +767,7 @@ foreach ($files as $splFileInfo) { // DB - $stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Inode, :Perms, :Owner, :ATime, :CTime, :MTime, :LinkTarget, :RealPath, :stat, :items, :newest, :gfi_type, :gfi_attr, :gfi_created, :Size, :Title, :PixelWidth, :PixelHeight, :Duration, :DateTimeOriginal, :Origin, :GPS, :Author, :spotlight, :kMDItemDateAdded, :kMDItemLastUsedDate, :kMDItemUseCount, :kMDItemContentModificationDate, :kMDItemContentType, :kMDItemCreator, :kMDItemFSCreatorCode, :kMDItemKind, :kMDItemFSTypeCode, :kMDItemUserTags, :kMDItemFSInvisible, :kMDItemNumberOfPages, :kMDItemPageHeight, :kMDItemPageWidth, :kMDItemWhereFroms, :kMDItemEncodingApplications, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :ProfileDescription, :BitDepth, :Compression, :Orientation, :LensType, :VideoFormat, :AudioFormat, :Tracks, :Profile, :Bitrate)"); + $stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Inode, :Perms, :Owner, :ATime, :CTime, :MTime, :LinkTarget, :RealPath, :stat, :items, :newest, :gfi_type, :gfi_attr, :gfi_created, :Size, :Title, :PixelWidth, :PixelHeight, :Duration, :DateTimeOriginal, :Origin, :GPS, :Author, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :ProfileDescription, :BitDepth, :Compression, :Orientation, :LensType, :VideoFormat, :AudioFormat, :Tracks, :Profile, :Bitrate)"); // Identify dir, file, link or bundle dir @@ -728,7 +813,7 @@ foreach ($files as $splFileInfo) { } - echo shortlabel($pathname,50,1); + echo shortlabel($pathname,50); // ------------------------------------------------ // @@ -754,7 +839,6 @@ foreach ($files as $splFileInfo) { $size = null; } $stmt->BindValue(":Size",@$size); - stringPrint(floor($size/1024)."k"); // ------------------------------------------------ // @@ -763,10 +847,8 @@ foreach ($files as $splFileInfo) { if ($type == "dir" || $type == "bundle" ) { $items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\..*' \) | wc -l 2>&1"))-1; $stmt->BindValue(":items",@$items); - stringPrint($items ? "ITEMS" : "items"); } else { $items = null; - stringPrint(" "); } // ------------------------------------------------ // @@ -776,10 +858,8 @@ foreach ($files as $splFileInfo) { if ($type == "dir") { $newest = @filemtime(chop(shell_exec("find ".$shellpath." -type f -not -path '*/\.*' -print0 | xargs -0 stat -f \"%m %N\" | sort -rn 2>&1 | head -1 | cut -f2- -d\" \""))); $stmt->BindValue(":newest",@$newest); - stringPrint($newest ? "NEWEST" : "newest"); } else { $newest = null; - stringPrint(" "); } // ------------------------------------------------ // @@ -800,63 +880,6 @@ foreach ($files as $splFileInfo) { $stmt->BindValue(":gfi_type",$writegfitype); $stmt->BindValue(":gfi_attr",@$gfi['attributes']); $stmt->BindValue(":gfi_created",strtotime($gfi['created'])); - - stringPrint("GFI"); - - // ------------------------------------------------ // - - // Spotlight - - $mdls = null; - $mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1"); - - if ($mdls != $pathname.": could not find ".$pathname.".\n") { - $parser = new plistParser(); - $spotlight = $parser->parseString($mdls); - //$stmt->BindValue(":spotlight",serialize($spotlight)); - $stmt->BindValue(":spotlight",$mdls); - } else { - $spotlight = array(); - $stmt->BindValue(":spotlight",null); - } - - stringPrint($mdls ? "MDLS" : "mdls"); - - unset($breakout, $schema, $item, $ready); - - $breakout[] = array ("kMDItemDateAdded", "date"); - $breakout[] = array ("kMDItemLastUsedDate", "date"); - $breakout[] = array ("kMDItemUseCount", 0); - $breakout[] = array ("kMDItemContentModificationDate", "date"); - $breakout[] = array ("kMDItemContentType", 0); - $breakout[] = array ("kMDItemCreator", 0); - $breakout[] = array ("kMDItemFSCreatorCode", 0); - $breakout[] = array ("kMDItemKind", 0); - $breakout[] = array ("kMDItemFSTypeCode", 0); - $breakout[] = array ("kMDItemUserTags", "array"); - $breakout[] = array ("kMDItemFSInvisible", 0); - $breakout[] = array ("kMDItemNumberOfPages", 0); - $breakout[] = array ("kMDItemPageHeight", 0); - $breakout[] = array ("kMDItemPageWidth", 0); - $breakout[] = array ("kMDItemWhereFroms", "array"); - $breakout[] = array ("kMDItemEncodingApplications", "array"); - - foreach ($breakout as $schema) { - if (!isset($spotlight[$schema[0]])) { - $stmt->BindValue(":".$schema[0],null); - continue; - } - if ($schema[1] === "date") { - $ready = strtotime($spotlight[$schema[0]]); - } elseif ($schema[1] === "array") { - $ready = serialize($spotlight[$schema[0]]); - } else { - $ready = $spotlight[$schema[0]]; - } - $stmt->BindValue(":".$schema[0],$ready); - } - - unset($breakout); // ------------------------------------------------ // @@ -869,119 +892,47 @@ foreach ($files as $splFileInfo) { $fetch_exif = @unserialize($dbp->query("SELECT tags FROM exiftool WHERE fid='".$fid."'")->fetch()[0]); is_array($fetch_exif) ? $yes_exif = 1 : $yes_exif = 0; $stmt->BindValue(":has_exif",$yes_exif); - stringPrint($yes_exif ? "EXIF" : "exif"); $fetch_media = @unserialize($dbp->query("SELECT info FROM mediainfo WHERE fid='".$fid."'")->fetch()[0]); is_array($fetch_media) ? $yes_media = 1 : $yes_media = 0; $stmt->BindValue(":has_mediainfo",$yes_media); - stringPrint($yes_media ? "MEDIA" : "media"); $yes_hash = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0]; $stmt->BindValue(":has_hash",$yes_hash); - stringPrint($yes_hash ? "HASH" : "hash"); - + $fetch_thumb = $dbp->query("SELECT * FROM thumbs WHERE fid='".$fid."'")->fetch(); if (@$fetch_thumb['relative_path']) { $stmt->BindValue(":thumb_filename",$fetch_thumb['relative_path']); $stmt->BindValue(":thumb_width",$fetch_thumb['width']); $stmt->BindValue(":thumb_height",$fetch_thumb['height']); - stringPrint("THUMB"); } else { $stmt->BindValue(":thumb_filename",null); - stringPrint("thumb"); } - $breakout[] = "ProfileDescription"; - $breakout[] = "BitDepth_BitsPerSample"; - $breakout[] = "Compression"; - $breakout[] = "Aperture,LightSource,WhiteBalance"; - $breakout[] = "Orientation"; - $breakout[] = "LensType,FocalLength"; - - $breakout['profile'] = "ProfileDescription"; - $breakout['bits'] = "BitDepth_BitsPerSample"; - $breakout['compression'] = "Compression"; - $breakout[] = "Aperture,LightSource,WhiteBalance"; - $breakout[] = "Orientation"; - $breakout[] = "LensType,FocalLength"; - -/* - -function parseConditionalItem($data, $item) { - if (strpos($item, "_")) { - $list = explode("_", $item); - foreach ($list as $piece) { - - // left off here - } - } - -function parseItem($data, $item) { - if (strpos($item, ",")) { - $list = explode(",", $item); - foreach ($list as $piece) { - $cleared[] = parseConditionalItem($piece); - } - } else { - $cleared[] = parseConditionalItem($item); - } - foreach ($cleared as $check) { - if (isset($data[$check])) { - $ready[] = $data[$check]; - } - } - if (@count($ready) > 1) { - return implode(", ",$ready); - } elseif (@count($ready) == 1) { - return $ready[0]; - } else { - return null; - } - } - -// left off here - - foreach ($breakout as $item) { - unset($ready); - if (strpos($item, ",")) { - $list = explode(",", $item) { - foreach ($list as $multi) { - if (isset($fetch_exif[$multi])) { - $ready[] = $fetch_exif[$multi]; - } - } - } elseif (strpos($item, "_")) { - $list = explode("_", $item) { - foreach ($list as $multi) { - if (isset($ready) { - continue; - } - if (isset($fetch_exif[$multi])) { - $ready[] = $fetch_exif[$multi]; - } + // ------------------------------------------------ // + // Breakouts + + + + + + + -*/ - - unset($breakout); - //print_r($fetch_exif); - //print_r($fetch_media); - - } + // ------------------------------------------------ // // Write to DB $stmt->execute(); - stringPrint("->DB"); // Set fileatime back to original value //if ($type != "link" && is_writable($pathname) && $p['fixatimes']) { // exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1"); - // stringPrint("touch"); // } // Double check stat for file against pre-run value @@ -1016,7 +967,7 @@ function parseItem($data, $item) { $message[] = "CTIME"; } - if (count($message)) { stringPrint("Changed: ".implode(", ", $message)); } + if (count($message)) { echo "\nChanged: ".implode(", ", $message)."\n"; } } @@ -1033,8 +984,10 @@ echo ProgressBar::finish(); echo "\n"; +if (file_exists($error_log_file)) { echo file_get_contents($error_log_file); } + $seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]); -$dbo->exec("UPDATE _walkwalk SET status='completed_in_".$seconds."'"); +$dbo->exec("UPDATE _skim SET status='completed_in_".$seconds."'"); echo "Finished in ".$seconds." seconds\n\n"; unset($dbo, $dbp, $files, $family, $fx); diff --git a/functions.php b/functions.php index 2ba273d..6e3c1ac 100755 --- a/functions.php +++ b/functions.php @@ -11,7 +11,7 @@ class ProgressBar { public static function display($message = null) { $string = "PROGRESS:".round((self::$done/self::$total)*100,2); if ($message) { - return "\t".$string."\n".$message; + return "\n".$string."\n".$message; } elseif (!strpos(__FILE__,".app")) { return "\r\033[K\r".$string; } else { diff --git a/web/rtc.php b/web/rtc.php index b69483a..6624929 100644 --- a/web/rtc.php +++ b/web/rtc.php @@ -3,9 +3,9 @@ @@ -14,6 +14,12 @@ img { width: 128px; height: 128px; } "; + // Show a view if (!is_readable($db_file)) { echo "can't read db file"; die; } - echo "<-"; + echo "index"; + + echo "
"; $dbo = new PDO("sqlite:".$db_file); $dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); $array = $dbo->query("SELECT children FROM family WHERE (rowid=2)")->fetch()['children']; - $array = unserialize($array); + $children = unserialize($array); //echo "
"; print_r($array); echo "
"; - foreach ($array as $key => $item) { + + echo "
"; + + echo "\n
"; + + foreach ($children as $key => $item) { - $row = $dbo->query("SELECT * FROM files WHERE (pid='".$item."')")->fetchAll()[0]; - //echo "
"; print_r($row); echo "
"; + $row_a = $dbo->query("SELECT * FROM files WHERE (pid='".$item."')")->fetchAll()[0]; - $icon = ""; + //$row_b = $dbo->query("SELECT * FROM mdls WHERE (pid='".$item."')")->fetchAll()[0]; + //$row_c = $dbo->query("SELECT * FROM milk WHERE (pid='".$item."')")->fetchAll()[0]; - echo "
".$icon."
".$row['Filename']."
"; + $icon = ""; + + echo "\n
".$icon."
".htmlentities($row_a['Filename'])."
"; echo "
"; } - echo "
"; + echo "\n"; } else { - foreach (glob("skim/*.bundle") as $skim) { - $dbs = glob($skim."/*.sqlite3"); - foreach ($dbs as $db) { - if (!strpos($db,"pool")) { - echo "".$db."
"; + // DB List + + $bundles = glob("skim/*.bundle"); + foreach ($bundles as $bundle) { + echo "

".pathinfo($bundle)['filename']."

"; + $dbs = glob($bundle."/*.sqlite3"); + foreach ($dbs as $db_file) { + if (!strpos($db_file,"pool")) { + echo "".pathinfo($db_file)['filename']." "; + $dbo = new PDO("sqlite:".$db_file); + echo $dbo->query("SELECT type FROM _skim WHERE (rowid=1)")->fetch()['type'].", "; + echo $dbo->query("SELECT passed_total FROM _skim WHERE (rowid=1)")->fetch()['passed_total']." files, "; + echo $dbo->query("SELECT status FROM _skim WHERE (rowid=1)")->fetch()['status']; + echo "
"; } } } diff --git a/web/rtc_legacy.php b/web/rtc_legacy.php index 6358b1d..9dcba4d 100644 --- a/web/rtc_legacy.php +++ b/web/rtc_legacy.php @@ -7,6 +7,7 @@ $browser_version = "0.4.5"; +require_once("togggle.php"); date_default_timezone_set("America/Los_Angeles"); $method = "id";