diff --git a/ProgressBar.php b/ProgressBar.php new file mode 100644 index 0000000..ba0e430 --- /dev/null +++ b/ProgressBar.php @@ -0,0 +1,342 @@ + "\r:message::padding:%.01f%% %2\$d/%3\$d Est: %4\$s. Elapsed: %5\$s [%6\$s]", + 'message' => 'Running', + 'size' => 30, + 'width' => null + ); + + /** + * Runtime options + */ + protected static $options = array(); + + /** + * How much have we done already + */ + protected static $done = 0; + + /** + * The format string used for the rendered status bar - see $defaults + */ + protected static $format; + + /** + * message to display prefixing the progress bar text + */ + protected static $message; + + /** + * How many chars to use for the progress bar itself. Not to be confused with $width + */ + protected static $size = 30; + + /** + * When did we start (timestamp) + */ + protected static $start; + + /** + * The width in characters the whole rendered string must fit in. defaults to the width of the + * terminal window + */ + protected static $width; + + /** + * What's the total number of times we're going to call set + */ + protected static $total; + + /** + * Show a progress bar, actually not usually called explicitly. Called by next() + * + * @param int $done what fraction of $total to set as progress uses internal counter if not passed + * + * @static + * @return string, the formatted progress bar prefixed with a carriage return + */ + public static function display($done = null) + { + if ($done) { + self::$done = $done; + } + + $now = time(); + + if (self::$total) { + $fractionComplete = (double) (self::$done / self::$total); + } else { + $fractionComplete = 0; + } + + $bar = floor($fractionComplete * self::$size); + $barSize = min($bar, self::$size); + + $barContents = str_repeat('=', $barSize); + if ($bar < self::$size) { + $barContents .= '>'; + $barContents .= str_repeat(' ', self::$size - $barSize); + } elseif ($fractionComplete > 1) { + $barContents .= '!'; + } else { + $barContents .= '='; + } + + $percent = number_format($fractionComplete * 100, 0); + + $elapsed = $now - self::$start; + if (self::$done) { + $rate = $elapsed / self::$done; + } else { + $rate = 0; + } + $left = self::$total - self::$done; + $etc = round($rate * $left, 2); + + if (self::$done) { + $etcNowText = '< 1 sec'; + } else { + $etcNowText = '???'; + } + $timeRemaining = self::humanTime($etc, $etcNowText); + $timeElapsed = self::humanTime($elapsed); + + $return = sprintf( + self::$format, + $percent, + self::$done, + self::$total, + $timeRemaining, + $timeElapsed, + $barContents + ); + + $width = strlen(preg_replace('@(?:\r|:\w+:)@', '', $return)); + + if (strlen(self::$message) > (self::$width - $width - 3)) { + $message = substr(self::$message, 0, (self::$width - $width - 4)) . '...'; + $padding = ''; + echo "\n" . strlen($return); + } else { + $message = self::$message; + $width += strlen($message); + $padding = str_repeat(' ', (self::$width - $width)); + } + + $return = str_replace(':message:', $message, $return); + $return = str_replace(':padding:', $padding, $return); + + return $return; + } + + /** + * reset internal state, and send a new line so that the progress bar text is "finished" + * + * @static + * @return string, a new line + */ + public static function finish() + { + self::reset(); + return "\n"; + } + + /** + * Increment the internal counter, and returns the result of display + * + * @param int $inc Amount to increment the internal counter + * @param string $message If passed, overrides the existing message + * + * @static + * @return string - the progress bar + */ + public static function next($inc = 1, $message = '') + { + self::$done += $inc; + + if ($message) { + self::$message = $message; + } + + return self::display(); + } + + /** + * Called by start and finish + * + * @param array $options array + * + * @static + * @return void + */ + public static function reset($options = array()) + { + $options = array_merge(self::$defaults, $options); + + if (empty($options['done'])) { + $options['done'] = 0; + } + if (empty($options['start'])) { + $options['start'] = time(); + } + if (empty($options['total'])) { + $options['total'] = 0; + } + + self::$done = $options['done']; + self::$format = $options['format']; + self::$message = $options['message']; + self::$size = $options['size']; + self::$start = $options['start']; + self::$total = $options['total']; + self::setWidth($options['width']); + } + + /** + * change the message to be used the next time the display method is called + * + * @param string $message the string to display + * + * @static + * @return void + */ + public static function setMessage($message = '') + { + self::$message = $message; + } + + /** + * change the total on a running progress bar + * + * @param int $total the new number of times we're expecting to run for + * + * @static + * @return void + */ + public static function setTotal($total = '') + { + self::$total = $total; + } + + /** + * Initialize a progress bar + * + * @param mixed $total number of times we're going to call set + * @param int $message message to prefix the bar with + * @param int $options overrides for default options + * + * @static + * @return string - the progress bar string with 0 progress + */ + public static function start($total = null, $message = '', $options = array()) + { + if ($message) { + $options['message'] = $message; + } + $options['total'] = $total; + $options['start'] = time(); + self::reset($options); + + return self::display(); + } + + /** + * Convert a number of seconds into something human readable like "2 days, 4 hrs" + * + * @param int $seconds how far in the future/past to display + * @param string $nowText if there are no seconds, what text to display + * + * @static + * @return string representation of the time + */ + protected static function humanTime($seconds, $nowText = '< 1 sec') + { + $prefix = ''; + if ($seconds < 0) { + $prefix = '- '; + $seconds = -$seconds; + } + + $days = $hours = $minutes = 0; + + if ($seconds >= 86400) { + $days = (int) ($seconds / 86400); + $seconds = $seconds - $days * 86400; + } + if ($seconds >= 3600) { + $hours = (int) ($seconds / 3600); + $seconds = $seconds - $hours * 3600; + } + if ($seconds >= 60) { + $minutes = (int) ($seconds / 60); + $seconds = $seconds - $minutes * 60; + } + $seconds = (int) $seconds; + + $return = array(); + + if ($days) { + $return[] = "$days days"; + } + if ($hours) { + $return[] = "$hours hrs"; + } + if ($minutes) { + $return[] = "$minutes mins"; + } + if ($seconds) { + $return[] = "$seconds secs"; + } + + if (!$return) { + return $nowText; + } + return $prefix . implode(array_slice($return, 0, 2), ', '); + } + + /** + * Set the width the rendered text must fit in + * + * @param int $width passed in options + * + * @static + * @return void + */ + protected static function setWidth($width = null) + { + if ($width === null) { + if (DIRECTORY_SEPARATOR === '/') { + $width = `tput cols`; + } + if ($width < 80) { + $width = 80; + } + } + self::$width = $width; + } +} diff --git a/Yuba.php b/Yuba.php index bb7a068..e1d0831 100755 --- a/Yuba.php +++ b/Yuba.php @@ -1,13 +1,37 @@ #!/usr/bin/php "; die; } + +// Check for bundle +if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); } +if (is_writable($zpath)) { $wopt_paranoid = 1; } else { $wopt_paranoid = 0; } + +$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle"; +if (!is_dir($bpath)) { mkdir($bpath); } +if (!is_dir($bpath."/thumbs")) { mkdir($bpath."/thumbs"); } + +$wopt_hash_limit = 1; // don't hash if exceeds in gigs, 0 for unlimited +$wopt_thumb_size = "512"; // Treat these directories as files $wopt_bundles = array( "app", @@ -40,428 +64,44 @@ $wopt_ignore = array( ".DS_Store", ".neofinder.abemeda.volinfo.xml" ); -if (in_array("-hash", $argv)) { $wopt_hash = 1; } else { $wopt_hash = 0; } - -$wopt_thumb_size = "256"; -$wopt_thumb_factor = $wopt_thumb_size/128; - -// Location of binaries used to collect metadata +$max_label = 50; +// Metadata tools $bin_gfi = "/Applications/Xcode.app/Contents/Developer/usr/bin/GetFileInfo"; $bin_mediainfo = "/opt/local/bin/mediainfo"; $bin_exiftool = "/opt/local/bin/exiftool"; +$bin_tq = "/opt/local/bin/ql-thumbnail"; +$bin_tv = "/opt/local/bin/vipsthumbnail"; +$bin_tf = "/usr/local/bin/ffmpegthumbnailer"; -// Functions - -function stringPrint($string) { - echo $string.@str_repeat(" ", (10-strlen($string))); - } - -function getWoptString() { - global $wopt_nodescend, $wopt_ignore, $wopt_hash, $wopt_mediainfo, $wopt_exiftool, $wopt_thumb_size, $wopt_thumb_factor, $wopt_tmpdir, $wopt_paranoid; - return array( array("nodescend", $wopt_nodescend), - array("ignore", $wopt_ignore), - array("hash", $wopt_hash), - array("mediainfo", $wopt_mediainfo), - array("exiftool", $wopt_exiftool), - array("thumb_size", $wopt_thumb_size), - array("thumb_factor", $wopt_thumb_factor), - array("wopt_paranoid", $wopt_paranoid) - ); - } - -class plistParser extends XMLReader { - public function parseString($string) { $this->XML($string); return $this->process(); } - private function process() { - $this->read(); - if($this->nodeType !== XMLReader::DOC_TYPE || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 2); } - if(!$this->next("plist") || $this->nodeType !== XMLReader::ELEMENT || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 3); } - $plist = array(); while($this->read()) { if($this->nodeType == XMLReader::ELEMENT) { $plist[] = $this->parse_node(); } } - if(count($plist) == 1 && $plist[0]) { return $plist[0]; } else { return $plist; } - } - private function parse_node() { - if($this->nodeType !== XMLReader::ELEMENT) return; - switch($this->name) { - case 'data': return base64_decode($this->getNodeText()); break; - case 'real': return floatval($this->getNodeText()); break; - case 'string': return $this->getNodeText(); break; - case 'integer': return intval($this->getNodeText()); break; - case 'date': return $this->getNodeText(); break; - case 'true': return true; break; - case 'false': return false; break; - case 'array': return $this->parse_array(); break; - case 'dict': return $this->parse_dict(); break; - default: throw new Exception(sprintf("Not a valid plist. %s is not a valid type", $this->name), 4); - } - } - private function parse_dict() { - $array = array(); $this->nextOfType(XMLReader::ELEMENT); - do { if($this->nodeType !== XMLReader::ELEMENT || $this->name !== "key") { if(!$this->next("key")) { return $array; } } $key = $this->getNodeText(); $this->nextOfType(XMLReader::ELEMENT); $array[$key] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); } - while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "dict")); return $array; - } - private function parse_array() { - $array = array(); $this->nextOfType(XMLReader::ELEMENT); - do { $array[] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); } - while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "array")); return $array; - } - private function getNodeText() { $string = $this->readString(); $this->nextOfType(XMLReader::END_ELEMENT); return $string; } - private function nextOfType() { $types = func_get_args(); $this->read(); while($this->nodeType && !(in_array($this->nodeType, $types))) { $this->read(); } } - private function isNodeOfTypeName($type, $name) { return $this->nodeType === $type && $this->name === $name; } - } - -function parseMediaInfo ($xml) { - $xml = simplexml_load_string($xml); - $data = array(); - $data['version'] = (string) $xml['version']; - foreach ($xml->File->track as $track) { - $trackType = strtolower($track['type']); - $trackId = isset($track['streamid']) ? $track['streamid'] : 1; - $trackId = (string)$trackId; - $trackData = []; - foreach ($track as $rawKey => $rawVal) { - $key = strtolower($rawKey); - $val = (string)$rawVal; - if ($key == 'stream_identifier') { continue; } - if (!array_key_exists($key, $trackData)) { - $trackData[$key] = array($val); - } elseif (!in_array($val, $trackData[$key])) { - $trackData[$key][] = $val; - } - } - if ($trackType == 'general') { - $data['file']['general'] = $trackData; - } else { - $data['file'][$trackType][$trackId] = $trackData; - } - } - return $data; - } - -function bashcolor($str,$fgcolor="white",$bgcolor=null) { - static $fgcolors = array('black' => '0;30', 'dark gray' => '1;30', 'blue' => '0;34', 'light blue' => '1;34', 'green' => '0;32', 'light green' => '1;32', 'cyan' => '0;36', 'light cyan' => '1;36', 'red' => '0;31', 'light red' => '1;31', 'purple' => '0;35', 'light purple' => '1;35', 'brown' => '0;33', 'yellow' => '1;33', 'light gray' => '0;37', 'white' => '1;37', 'underline' => '4'); - static $bgcolors = array('black' => '40', 'red' => '41', 'green' => '42', 'yellow' => '43', 'blue' => '44', 'magenta' => '45', 'cyan' => '46', 'light gray' => '47'); - $out=""; - if (!isset($fgcolors[$fgcolor])) { $fgcolor='white'; } - if (!isset($bgcolors[$bgcolor])) { $bgcolor=null; } - if ($fgcolor) { $out .= "\033[{$fgcolors[$fgcolor]}m"; } - if ($bgcolor) { $out .= "\033[{$bgcolors[$bgcolor]}m"; } - $out .= $str."\033[0m"; - return $out; - } - -// Path arguments - -$zpath = realpath($argv[1]); -if (!is_dir($zpath)) { echo "Usage: walk "; die; } -if (isset($argv[2]) && is_dir($argv[2])) { - $dbprefix = realpath($argv[2]); - } else { - $dbprefix = "."; - } - -if (is_writable($zpath)) { $wopt_paranoid = 1; } else { $wopt_paranoid = 0; } - -// File checks - -$stamp = date("Y-m-d_H-i-s", time()); -$wopt_tmpdir = "/tmp/WalkWalk_".$stamp."/"; -if (!is_dir($wopt_tmpdir)) { mkdir($wopt_tmpdir); } -$base = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); -if (!$base) { $base = "root"; } -$dbfile = $dbprefix."/".$stamp."_".$base.".sqlite3"; -if (file_exists($dbfile)) { echo "File \"".$dbfile."\" already exists!"; die; } - -// Banner - -echo "Yuba ".$version."\n"; -echo "-----------------------------------------------\n"; -$banner = $zpath." -> ".$dbfile; -echo $banner."\n"; -echo str_repeat("-", strlen($banner))."\n"; - -// Disk info - -$host = gethostname(); -$disks = shell_exec("diskutil list 2>&1"); - -if (substr($zpath, 0, 9) != "/Volumes/") { - $zbase = "/"; - } else { - $zparts = explode("/", $zpath); - $zbase = "/Volumes/".$zparts[2]; - } - -$diskutil = shell_exec("diskutil info ".$zbase." 2>&1"); -$getstats = array( "Volume Name", - "Protocol", - "Volume UUID", - "Device Location", - "Volume Total Space", - "Volume Available Space", - "Level Type" - ); -foreach ($getstats as $stat) { - preg_match("/(".$stat.":)(.*)(\n)/",$diskutil,$matches); - if (isset($matches[2])) { - if (substr($stat, -5, 5) == "Space") { - $pieces = explode(" ", trim($matches[2])); - $summary = $pieces[0]." ".$pieces[1]; - $stats[$stat] = $summary; - } else { - $stats[$stat] = trim($matches[2]); - } - } - } - -$dstats = serialize($stats); -if ($zpath == "/") { - $type = "Startup disk"; - } elseif (strtolower($zpath) == strtolower("/Volumes/".$stats["Volume Name"])) { - if ($stats["Protocol"] == "Disk Image") { - $type = "Disk image"; - } else { - $type = "External disk"; - } - } else { - $type = "Folder"; - } - -$profile = shell_exec("system_profiler SPHardwareDataType SPStorageDataType SPThunderboltDataType SPUSBDataType 2>&1"); -$qlmanage = shell_exec("qlmanage -m 2>&1"); -$sysvers = shell_exec("sw_vers 2>&1"); - -// Database - -// use mysql? -//$dbo = new PDO("mysql:dbname=testdb;host=127.0.0.1", $user, $pass); - -$dbo = new PDO("sqlite:".$dbfile); -$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); - -$dbo->query("PRAGMA page_size = 4096"); -$dbo->query("PRAGMA cache_size = 10000"); -$dbo->query("PRAGMA locking_mode = EXCLUSIVE"); -$dbo->query("PRAGMA synchronous = NORMAL"); -$dbo->query("PRAGMA journal_mode = WAL"); - -$dbo->exec("CREATE TABLE files ( - id INTEGER PRIMARY KEY, - parent INTEGER, - Pathname TEXT, - Path TEXT, - Filename TEXT, - Extension TEXT, - Type TEXT, - items INTEGER, - newest INTEGER, - stat TEXT, - LinkTarget TEXT, - RealPath TEXT, - Inode INTEGER, - Size INTEGER, - Perms INTEGER, - Owner TEXT, - ATime INTEGER, - MTime INTEGER, - CTime INTEGER, - gfi_type TEXT, - gfi_attr TEXT, - gfi_created TEXT, - hash TEXT, - tinfo TEXT - )"); - -$dbo->exec("CREATE TABLE mdls ( - id INTEGER PRIMARY KEY, - hasmeta INTEGER, - DateAdded TEXT, - ContentType TEXT, - Creator TEXT, - Kind TEXT, - UserTags TEXT, - FSInvisible INTEGER, - PixelWidth INTEGER, - PixelHeight INTEGER, - spotlight TEXT - )"); - -$dbo->exec("CREATE TABLE metadata ( - id INTEGER PRIMARY KEY, - duration TEXT, - mediainfo TEXT, - exiftool TEXT - )"); - -$dbo->exec("CREATE TABLE thumbs ( - id INTEGER PRIMARY KEY, - thumb BLOB - )"); - -$dbo->exec("CREATE TABLE _walkwalk ( - version TEXT, - opts TEXT, - host TEXT, - zpath TEXT, - type TEXT, - nodescended INTEGER, - ignored INTEGER, - stats TEXT, - qlmanage TEXT, - sysvers TEXT, - diskutil TEXT, - disks TEXT, - profile TEXT, - status TEXT - )"); - -$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :zpath, :type, :nodescended, :ignored, :stats, :qlmanage, :sysvers, :diskutil, :disks, :profile, :status)"); -$stmt->BindValue(":version",$version." (".posix_getuid().")"); -$stmt->BindValue(":opts",serialize(getWoptString())); -$stmt->BindValue(":host",$host); -$stmt->BindValue(":zpath",$zpath); -$stmt->BindValue(":type",$type); -$stmt->BindValue(":nodescended",null); -$stmt->BindValue(":ignored",null); -$stmt->BindValue(":stats",$dstats); -$stmt->BindValue(":qlmanage",$qlmanage); -$stmt->BindValue(":sysvers",$sysvers); -$stmt->BindValue(":diskutil",$diskutil); -$stmt->BindValue(":disks",$disks); -$stmt->BindValue(":profile",$profile); -$stmt->BindValue(":status","aborted"); -$stmt->execute(); - -// Iterate - -$nodescended = 0; -$ignored = 0; -$files = new RecursiveIteratorIterator( - new RecursiveCallbackFilterIterator( - new RecursiveDirectoryIterator( - // start in parent dir to include self - dirname($zpath), - RecursiveDirectoryIterator::SKIP_DOTS - ), - function ($current, $key, $iterator) use ($wopt_ignore, $wopt_nodescend) { - global $nodescended, $ignored, $zpath; - $clean = true; - // ensure we don't traverse zpath siblings - if ($zpath != "/" && (substr($current->getRealpath(), 0, strlen($zpath)+1) != $zpath."/") && ($current->getRealpath() != $zpath)) { - $clean = false; - } - // filenames to ignore - if (is_array($wopt_ignore)) { - foreach ($wopt_ignore as $wildcard) { - if (fnmatch($wildcard, $current->getFilename())) { - $clean = false; - $ignored++; - echo "\nSkipping: ".$current->getRealpath()."\n\n"; - } - } - } - // directories to ignore - if (is_array($wopt_nodescend)) { - foreach ($wopt_nodescend as $wildcard) { - if (fnmatch($wildcard, $current->getPath())) { - $clean = false; - $nodescended++; - echo "\nNodescending: ".$current->getRealpath()."\n\n"; - } - } - } - return $clean; - } - ), - RecursiveIteratorIterator::SELF_FIRST, - RecursiveIteratorIterator::CATCH_GET_CHILD - ); - -// Debug input array - -/* -foreach ($files as $splFileInfo) { - echo "====================================================================================\n"; - print_r($splFileInfo); - echo "====================================================================================\n"; - echo "getRealPath = ".$splFileInfo->getRealPath()."\n"; - echo "getPathname = ".$splFileInfo->getPathname()."\n"; - echo "isLink = ".$splFileInfo->isLink()."\n"; - if ($splFileInfo->isLink()) { - echo "getLinkTarget = ".$splFileInfo->getLinkTarget()."\n"; - } else { - echo "getLinkTarget = N/A\n"; - } - echo "getType = ".$splFileInfo->getType()." (".filetype($splFileInfo->getPathname()).")\n"; - echo "isDir = ".$splFileInfo->isDir()."\n"; - echo "isFile = ".$splFileInfo->isFile()."\n"; - echo "-----------------------------------------------------------------------\n"; - echo "getPath = ".$splFileInfo->getPath()."\n"; - echo "getFilename = ".$splFileInfo->getFilename()."\n"; - echo "getBasename = ".$splFileInfo->getBasename()." (".basename($splFileInfo->getPathname()).")\n"; - echo "getExtension = ".$splFileInfo->getExtension()."\n"; - echo "-----------------------------------------------------------------------\n"; - if (!$splFileInfo->isLink()) { - echo "getATime = ".$splFileInfo->getAtime()." (".fileatime($splFileInfo->getPathname()).")\n"; - echo "getMTime = ".$splFileInfo->getMtime()." (".filemtime($splFileInfo->getPathname()).")\n"; - echo "getCTime = ".$splFileInfo->getCtime()." (".filectime($splFileInfo->getPathname()).")\n"; - echo "getInode = ".$splFileInfo->getInode()." (".fileinode($splFileInfo->getPathname()).")\n"; - echo "getPerms = ".$splFileInfo->getPerms()." (".fileperms($splFileInfo->getPathname()).")\n"; - echo "getGroup = ".$splFileInfo->getGroup()." (".filegroup($splFileInfo->getPathname()).")\n"; - echo "getOwner = ".$splFileInfo->getOwner()." (".fileowner($splFileInfo->getPathname()).")\n"; - echo "getSize = ".$splFileInfo->getSize()." (".filesize($splFileInfo->getPathname()).")\n"; - } - } -die; -*/ - -// Check perms - -if (posix_getuid()) { - - echo bashcolor("You are not root. Checking file readability: ", "red"); - - echo "\n"; - - $oops = 0; - foreach ($files as $splFileInfo) { - $path = $splFileInfo->getRealPath(); - if (!is_readable($path)) { - $oops = 1; - echo "x"; - } else { - echo "."; - } - } - - echo "\n\n"; - - if ($oops) { - echo "Some files could not be read. Continue? (Y/n)"; - $line = trim(fgets(fopen("php://stdin","r"))); - $line = $line ?: "y"; - if($line != "y"){ - echo "Exiting!\n"; die; - } - } - - } else { - - echo bashcolor("Running as root. Some QuickLook plugins may not be available.", "red"); - echo "\n"; - - } - -$fixatimes = 0; -if ($wopt_paranoid) { - - echo bashcolor("\nFilesystem is writable. You can choose:\n(c) Preserve ctimes (default)\n(a) Preserve atimes\n", "purple"); - $line = trim(fgets(fopen("php://stdin","r"))) ?: "c"; - if ($line == "a") { $fixatimes = 1; } - - } - -// Filetypes for special handling +// Media extensions +////////////////////////////////////////// +$t_files['ffmpeg'] = array( "mkv", + "avi", + "mpeg", + "mpg", + "vob", + "mp4", + "m4v", + "m2v", + "m2ts", + "asf", + "wmv", + "rm", + "divx", + "fla", + "flv", + "webm" ); + +$t_files['vips'] = array( "jpg", + "jpeg", + "tif", + "tiff", + "gif", + "psd", + "png" ); + $m_files = array( "mkv", "ogg", "avi", @@ -554,297 +194,638 @@ $e_files = array( "ai", "xls", "xlsx", "xmp", - "zip" ); + "zip" ); foreach ($e_files as $ext) { $e_files[] = strtoupper($ext); } foreach ($m_files as $ext) { $m_files[] = strtoupper($ext); } +foreach ($t_files['ffmpeg'] as $ext) { $t_files['ffmpeg'][] = strtoupper($ext); } +foreach ($t_files['vips'] as $ext) { $t_files['vips'][] = strtoupper($ext); } -// Inserts +// Functions +////////////////////////////////////////// -foreach ($files as $splFileInfo) { +/* +function getParents($zpath, $pathname) { + $path = dirname($pathname); + $parts = explode("/",trim(substr($path,strlen(basename($zpath))),"/")); + foreach ($parts as $index => $part) { + $parents[] = array($part, md5($zpath."/".implode("/",array_slice($parts, 0, $index+1)))); + } + return $parents; + } +*/ - $type = $splFileInfo->getType(); - if ($type == "dir") { - foreach ($wopt_bundles as $bundle) { - $check = ".".$bundle; - if (substr($splFileInfo->getFilename(), -(strlen($check)), strlen($check)) == $check) { $type = "bundle"; } - } - } - - if ($type != "link") { $atime = $splFileInfo->getATime(); } - $pathname = $splFileInfo->getPathname(); - $path = $splFileInfo->getPath(); - $filename = $splFileInfo->getFilename(); - $extension = $splFileInfo->getExtension(); - $shellpath = escapeshellarg($pathname); - - echo bashcolor("\n".$pathname."\n","blue"); - - if (!$type) { echo "\nBREAK: can't determine type of ".$pathname; die; } - - if ($type != "link") { - $stat = chop(@shell_exec("stat -x ".$shellpath." 2>&1")); +function shortlabel($pathname, $max) { + $basename = basename($pathname); + $suffix = "(...).".pathinfo($basename,PATHINFO_EXTENSION); + if (strlen($basename) > $max) { + return substr($basename, 0, ($max-strlen($suffix))).$suffix; } else { - $stat = null; - } - - if ($type != "link" && $wopt_paranoid) { - $pre_access = null; - $pre_modify = null; - $pre_change = null; - foreach (explode("\n", $stat) as $line) { - $check = substr($line, 0, 6); - if ($check == "Access") { $pre_access = $line; } - if ($check == "Modify") { $pre_modify = $line; } - if ($check == "Change") { $pre_change = $line; } - } + return $basename; } + } - // Determine ID of parent dir by querying database +function human_filesize($bytes, $decimals = 2) { + $size = array('B','kB','MB','GB','TB','PB','EB','ZB','YB'); + $factor = floor((strlen($bytes) - 1) / 3); + return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$size[$factor]; + } - $parent = $dbo->query("SELECT id FROM files WHERE (Pathname='".str_replace("'", "''", $path)."')")->fetch()['id']; - stringPrint("parent"); +function stringPrint($string) { + echo $string.@str_repeat(" ", (10-strlen($string))); + } - // Gather file attributes - - $stmt = $dbo->prepare("INSERT INTO files VALUES (:id, :parent, :Pathname, :Path, :Filename, :Extension, :Type, :items, :newest, :stat, :LinkTarget, :RealPath, :Inode, :Size, :Perms, :Owner, :ATime, :MTime, :CTime, :gfi_type, :gfi_attr, :gfi_created, :hash, :tinfo)"); - - if ($type == "dir") { - $size = trim(shell_exec("du -ks ".$shellpath." | cut -f1"))*1024; - } elseif ($type == "file" || $type == "bundle") { - $size = $splFileInfo->getSize(); - } else { - $size = null; - } - $stmt->BindValue(":Size",@$size); - stringPrint(floor($size/1024)."k"); - - if ($parent) { - $stmt->BindValue(":parent",$parent); - } else { - $stmt->BindValue(":parent",0); - } - - stringPrint("items"); - - if ($type == "dir" || $type == "bundle" ) { - $items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\..*' \) | wc -l 2>&1"))-1; - } else { - $items = null; - } - - $stmt->BindValue(":items",@$items); - - stringPrint("newest"); - - if ($type == "dir") { - $newest = filemtime(chop(shell_exec("find ".$shellpath." -type f -not -path '*/\.*' -print0 | xargs -0 stat -f \"%m %N\" | sort -rn | head -1 | cut -f2- -d\" \""))); - } else { - $newest = null; - } - - $stmt->BindValue(":newest",@$newest); - - $stmt->BindValue(":stat",@$stat); - - $stmt->BindValue(":Pathname",$pathname); - $stmt->BindValue(":Path",$path); - $stmt->BindValue(":Filename",$filename); - $stmt->BindValue(":Extension",$extension); - $stmt->BindValue(":Type",$type); - - if ($type == "link") { - - $stmt->BindValue(":LinkTarget",$splFileInfo->getLinkTarget()); - $stmt->BindValue(":RealPath",$splFileInfo->getRealPath()); - - } else { - - $stmt->BindValue(":Inode",$splFileInfo->getInode()); - $stmt->BindValue(":Perms",$splFileInfo->getPerms()); - $stmt->BindValue(":Owner",$splFileInfo->getOwner().":".$splFileInfo->getGroup()); - - $stmt->BindValue(":ATime",$atime); - $stmt->BindValue(":CTime",$splFileInfo->getCTime()); - $stmt->BindValue(":MTime",$splFileInfo->getMTime()); - - } - - stringPrint("attr"); +function getWoptString() { + global $wopt_bundles, $wopt_ignore, $wopt_hash, $wopt_hash_limit, $wopt_mediainfo, $wopt_exiftool, $wopt_thumbs, $wopt_thumb_size, $wopt_paranoid; + return array( array("bundles", $wopt_bundles), + array("ignore", $wopt_ignore), + array("hash", $wopt_hash), + array("wopt_hash_limit", $wopt_hash_limit), + array("mediainfo", $wopt_mediainfo), + array("exiftool", $wopt_exiftool), + array("thumbs", $wopt_thumbs), + array("thumb_size", $wopt_thumb_size), + array("wopt_paranoid", $wopt_paranoid), + ); + } - $gfiparts = explode("\n", chop(shell_exec($bin_gfi." -P ".$shellpath." 2>&1"))); - if (is_array($gfiparts)) { - foreach ($gfiparts as $line) { - list($label, $value) = explode(": ", $line); - $gfi[$label] = isset($value) ? trim($value,"\"") : null; - } +class plistParser extends XMLReader { + public function parseString($string) { $this->XML($string); return $this->process(); } + private function process() { + $this->read(); + if($this->nodeType !== XMLReader::DOC_TYPE || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 2); } + if(!$this->next("plist") || $this->nodeType !== XMLReader::ELEMENT || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 3); } + $plist = array(); while($this->read()) { if($this->nodeType == XMLReader::ELEMENT) { $plist[] = $this->parse_node(); } } + if(count($plist) == 1 && $plist[0]) { return $plist[0]; } else { return $plist; } } - - $writegfitype = @$gfi['type'].":".@$gfi['creator']; - if ($writegfitype == "\\0\\0\\0\\0:\\0\\0\\0\\0" || $writegfitype == ":") { $writegfitype = null; } - - $stmt->BindValue("gfi_type",$writegfitype); - $stmt->BindValue("gfi_attr",@$gfi['attributes']); - $stmt->BindValue("gfi_created",strtotime($gfi['created'])); - - stringPrint("gfi"); - - if ($wopt_hash && $type != "link") { - $stmt->BindValue(":hash",md5_file($pathname)); - StringPrint("hash"); - } else { - $stmt->BindValue(":hash",null); - StringPrint("(x)hash"); + private function parse_node() { + if($this->nodeType !== XMLReader::ELEMENT) return; + switch($this->name) { + case 'data': return base64_decode($this->getNodeText()); break; + case 'real': return floatval($this->getNodeText()); break; + case 'string': return $this->getNodeText(); break; + case 'integer': return intval($this->getNodeText()); break; + case 'date': return $this->getNodeText(); break; + case 'true': return true; break; + case 'false': return false; break; + case 'array': return $this->parse_array(); break; + case 'dict': return $this->parse_dict(); break; + default: throw new Exception(sprintf("Not a valid plist. %s is not a valid type", $this->name), 4); + } } + private function parse_dict() { + $array = array(); $this->nextOfType(XMLReader::ELEMENT); + do { if($this->nodeType !== XMLReader::ELEMENT || $this->name !== "key") { if(!$this->next("key")) { return $array; } } $key = $this->getNodeText(); $this->nextOfType(XMLReader::ELEMENT); $array[$key] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); } + while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "dict")); return $array; + } + private function parse_array() { + $array = array(); $this->nextOfType(XMLReader::ELEMENT); + do { $array[] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); } + while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "array")); return $array; + } + private function getNodeText() { $string = $this->readString(); $this->nextOfType(XMLReader::END_ELEMENT); return $string; } + private function nextOfType() { $types = func_get_args(); $this->read(); while($this->nodeType && !(in_array($this->nodeType, $types))) { $this->read(); } } + private function isNodeOfTypeName($type, $name) { return $this->nodeType === $type && $this->name === $name; } + } - $thumb = $wopt_tmpdir.$filename.".png"; - /* - // insane workaround for insane apple bug - if (@pathinfo($pathname)['extension'] == "mp3" | "MP3" | "m4a" | "M4A" | "m4b" | "M4B" ) { - $zprefix = "sudo qlmanage"; - } else { - $zprefix = "qlmanage"; - } - */ - $zprefix = "sudo qlmanage"; // prevent hang - @exec($zprefix." -t -f ".$wopt_thumb_factor." -o ".$wopt_tmpdir." ".$shellpath." 2>&1"); - stringPrint("thumb"); - if ($size && !file_exists($thumb) && (in_array($extension, $m_files) || in_array($extension, $e_files))) { - @exec("ffmpegthumbnailer -i ".$shellpath." -o \"".$thumb."\" -s ".$wopt_thumb_size." -c png 2>&1"); - stringPrint("fthumb"); - } else { - stringPrint("(x)fthumb"); - } - if (file_exists($thumb) && filesize($thumb)) { - $stmt->BindValue(":tinfo",serialize(getimagesize($thumb))); - } - - $stmt->execute(); - stringPrint("->files"); - - // Gather spotlight metadata - - $stmt = $dbo->prepare("INSERT INTO mdls VALUES (:id, :hasmeta, :DateAdded, :ContentType, :Creator, :Kind, :UserTags, :FSInvisible, :PixelWidth, :PixelHeight, :spotlight)"); - $mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1"); - stringPrint("mdls"); - - if ($mdls != $pathname.": could not find ".$pathname.".\n") { - - $stmt->BindValue(":hasmeta",1); - $parser = new plistParser(); - $spotlight = $parser->parseString($mdls); - - $stmt->BindValue(":DateAdded",@strtotime($spotlight['kMDItemDateAdded'])); - $stmt->BindValue(":ContentType",@$spotlight['kMDItemContentType']); - $stmt->BindValue(":Creator",@$spotlight['kMDItemCreator']); - $stmt->BindValue(":Kind",@$spotlight['kMDItemKind']); - if (isset($spotlight['kMDItemUserTags'])) { - $stmt->BindValue(":UserTags",serialize($spotlight['kMDItemUserTags'])); - } - $stmt->BindValue(":FSInvisible",@$spotlight['kMDItemFSInvisible']); - $stmt->BindValue(":PixelWidth",@$spotlight['kMDItemPixelWidth']); - $stmt->BindValue(":PixelHeight",@$spotlight['kMDItemPixelHeight']); - $stmt->BindValue(":spotlight",serialize($spotlight)); - - } else { - $stmt->BindValue(":hasmeta",0); - } - $stmt->execute(); - stringPrint("->mdls"); - - // Gather external metadata - - $stmt = $dbo->prepare("INSERT INTO metadata VALUES (:id, :duration, :mediainfo, :exiftool)"); - - if ($type != "dir" && in_array($extension, $m_files)) { - $minfo = parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")); - if ($minfo['file']['general']['duration'][0]) { - $stmt->BindValue(":duration",$minfo['file']['general']['duration'][0]); +function parseMediaInfo ($xml) { + $xml = simplexml_load_string($xml); + $data = array(); + $data['version'] = (string) $xml['version']; + foreach ($xml->File->track as $track) { + $trackType = strtolower($track['type']); + $trackId = isset($track['streamid']) ? $track['streamid'] : 1; + $trackId = (string)$trackId; + $trackData = []; + foreach ($track as $rawKey => $rawVal) { + $key = strtolower($rawKey); + $val = (string)$rawVal; + if ($key == 'stream_identifier') { continue; } + if (!array_key_exists($key, $trackData)) { + $trackData[$key] = array($val); + } elseif (!in_array($val, $trackData[$key])) { + $trackData[$key][] = $val; + } + } + if ($trackType == 'general') { + $data['file']['general'] = $trackData; } else { - $stmt->BindValue(":duration",null); + $data['file'][$trackType][$trackId] = $trackData; } - $stmt->BindValue(":mediainfo",serialize($minfo)); - stringPrint("minfo"); + } + return $data; + } + +function bashcolor($str,$fgcolor="white",$bgcolor=null) { + static $fgcolors = array('black' => '0;30', 'dark gray' => '1;30', 'blue' => '0;34', 'light blue' => '1;34', 'green' => '0;32', 'light green' => '1;32', 'cyan' => '0;36', 'light cyan' => '1;36', 'red' => '0;31', 'light red' => '1;31', 'purple' => '0;35', 'light purple' => '1;35', 'brown' => '0;33', 'yellow' => '1;33', 'light gray' => '0;37', 'white' => '1;37', 'underline' => '4'); + static $bgcolors = array('black' => '40', 'red' => '41', 'green' => '42', 'yellow' => '43', 'blue' => '44', 'magenta' => '45', 'cyan' => '46', 'light gray' => '47'); + $out=""; + if (!isset($fgcolors[$fgcolor])) { $fgcolor='white'; } + if (!isset($bgcolors[$bgcolor])) { $bgcolor=null; } + if ($fgcolor) { $out .= "\033[{$fgcolors[$fgcolor]}m"; } + if ($bgcolor) { $out .= "\033[{$bgcolors[$bgcolor]}m"; } + $out .= $str."\033[0m"; + return $out; + } + +// Disk info +////////////////////////////////////////// + +$host = gethostname(); +$disks = shell_exec("diskutil list 2>&1"); + +if (substr($zpath, 0, 9) != "/Volumes/") { + $zbase = "/"; + } else { + $zparts = explode("/", $zpath); + $zbase = "/Volumes/".$zparts[2]; + } + +$diskutil = shell_exec("diskutil info ".$zbase." 2>&1"); +$getstats = array( "Volume Name", + "Protocol", + "Volume UUID", + "Device Location", + "Volume Total Space", + "Volume Available Space", + "Level Type" + ); +foreach ($getstats as $stat) { + preg_match("/(".$stat.":)(.*)(\n)/",$diskutil,$matches); + if (isset($matches[2])) { + if (substr($stat, -5, 5) == "Space") { + $pieces = explode(" ", trim($matches[2])); + $summary = $pieces[0]." ".$pieces[1]; + $stats[$stat] = $summary; + } else { + $stats[$stat] = trim($matches[2]); + } + } + } + +$dstats = serialize($stats); +if ($zpath == "/") { + $type = "Startup disk"; + } elseif (strtolower($zpath) == strtolower("/Volumes/".$stats["Volume Name"])) { + if ($stats["Protocol"] == "Disk Image") { + $type = "Disk image"; } else { - $stmt->BindValue(":duration",null); - $stmt->BindValue(":mediainfo",null); - stringPrint("(x)info"); + $type = "External disk"; } + } else { + $type = "Folder"; + } + +$profile = shell_exec("system_profiler SPHardwareDataType SPStorageDataType SPThunderboltDataType SPUSBDataType 2>&1"); +$qlmanage = shell_exec("qlmanage -m 2>&1"); +$sysvers = shell_exec("sw_vers 2>&1"); + +// Database +////////////////////////////////////////// + +$stamp = date("Y-m-d_H-i-s", time()); + +$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite"); +$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + +/* +$dbo->query("PRAGMA page_size = 4096"); +$dbo->query("PRAGMA cache_size = 10000"); +$dbo->query("PRAGMA locking_mode = EXCLUSIVE"); +$dbo->query("PRAGMA synchronous = NORMAL"); +$dbo->query("PRAGMA journal_mode = WAL"); +*/ + +$dbo->exec("CREATE TABLE _walkwalk ( + version TEXT, + opts TEXT, + host TEXT, + uid INTEGER, + zpath TEXT, + bpath TEXT, + type TEXT, + passed_file INTEGER, + passed_dir INTEGER, + passed_link INTEGER, + passed_total INTEGER, + nodescended INTEGER, + ignored INTEGER, + dupes INTEGER, + stats TEXT, + qlmanage TEXT, + sysvers TEXT, + diskutil TEXT, + disks TEXT, + profile TEXT, + status TEXT + )"); + +$dbo->exec("CREATE TABLE family ( + pid TEXT, + fid TEXT, + children TEXT + )"); + +$dbo->exec("CREATE TABLE files ( + pid TEXT, + fid TEXT, + Pathname TEXT, + Path TEXT, + Filename TEXT, + Extension TEXT, + Type TEXT, + items INTEGER, + newest INTEGER, + stat TEXT, + LinkTarget TEXT, + RealPath TEXT, + Inode INTEGER, + Size INTEGER, + Perms INTEGER, + Owner TEXT, + ATime INTEGER, + MTime INTEGER, + CTime INTEGER, + gfi_type TEXT, + gfi_attr TEXT, + gfi_created TEXT, + hash TEXT, + tinfo TEXT, + hasmeta INTEGER, + DateAdded TEXT, + ContentType TEXT, + Creator TEXT, + Kind TEXT, + UserTags TEXT, + FSInvisible INTEGER, + PixelWidth INTEGER, + PixelHeight INTEGER, + spotlight TEXT, + duration TEXT, + mediainfo TEXT, + exiftool TEXT + )"); + +$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :profile, :status)"); +$stmt->BindValue(":version",$version); +$stmt->BindValue(":opts",serialize(getWoptString())); +$stmt->BindValue(":host",$host); +$stmt->BindValue(":uid",posix_getuid()); +$stmt->BindValue(":zpath",$zpath); +$stmt->BindValue(":bpath",$bpath); +$stmt->BindValue(":type",$type); +$stmt->BindValue(":stats",$dstats); +$stmt->BindValue(":qlmanage",$qlmanage); +$stmt->BindValue(":sysvers",$sysvers); +$stmt->BindValue(":diskutil",$diskutil); +$stmt->BindValue(":disks",$disks); +$stmt->BindValue(":profile",$profile); +$stmt->BindValue(":status","aborted"); +$stmt->execute(); + +// Iterator +////////////////////////////////////////// + +$passed_file = $passed_dir = $passed_link = $nodescended = $ignored = 0; +$files = new RecursiveIteratorIterator( + new RecursiveCallbackFilterIterator( + new RecursiveDirectoryIterator( + // start in parent dir to include self + dirname($zpath), + RecursiveDirectoryIterator::SKIP_DOTS + ), + function ($current, $key, $iterator) use ($wopt_ignore, $wopt_nodescend) { + global $nodescended, $ignored, $passed_file, $passed_dir, $passed_link, $zpath; + $clean = true; + // ensure we don't traverse zpath siblings + if ($zpath != "/" && (substr($current->getRealpath(), 0, strlen($zpath)+1) != $zpath."/") && ($current->getRealpath() != $zpath)) { + $clean = false; + } + // filenames to ignore + if (is_array($wopt_ignore)) { + foreach ($wopt_ignore as $wildcard) { + if (fnmatch($wildcard, $current->getFilename())) { + $clean = false; + $ignored++; + //echo "\nSkipping: ".$current->getRealpath()."\n\n"; + } + } + } + // directories to ignore + if (is_array($wopt_nodescend)) { + foreach ($wopt_nodescend as $wildcard) { + if (fnmatch($wildcard, $current->getPath())) { + $clean = false; + $nodescended++; + //echo "\nNodescending: ".$current->getRealpath()."\n\n"; + } + } + } + if ($clean) { + if ($current->getType() == "file") { + $passed_file++; + } elseif ($current->getType() == "dir") { + $passed_dir++; + } elseif ($current->getType() == "link") { + $passed_link++; + } + } + return $clean; + } + ), + RecursiveIteratorIterator::SELF_FIRST, + RecursiveIteratorIterator::CATCH_GET_CHILD + ); + +// Banner +////////////////////////////////////////// + +echo "Yuba ".$version."\n"; +echo "-----------------------------------------------\n"; +$banner = $zpath." -> ".$bpath; +echo $banner."\n"; +echo str_repeat("-", strlen($banner))."\n"; + +// Permissions +////////////////////////////////////////// + +/* +if (posix_getuid()) { - if ($type != "dir" && $type != "link" && in_array($extension, $e_files)) { - $stmt->BindValue(":exiftool",serialize(eval("return ".`$bin_exiftool -php $shellpath`))); - stringPrint("etool"); - } else { - $stmt->BindValue(":exiftool",null); - stringPrint("(x)etool"); - } - $stmt->execute(); - stringPrint("->meta"); - - // Gather thumbnail - - $stmt = $dbo->prepare("INSERT INTO thumbs VALUES (:id, :thumb)"); - if (file_exists($thumb) && filesize($thumb)) { - $stmt->BindValue(":thumb",file_get_contents($thumb)); - } else { - $stmt->BindValue(":thumb",null); - } - $stmt->execute(); - stringPrint("->thumb"); - - // Set fileatime back to original value - - if ($type != "link" && is_writable($pathname) && $fixatimes) { - @exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1"); - stringPrint("touch"); - } + echo bashcolor("You are not root. Checking file readability: ", "red"); echo "\n"; - // Double check stat for file against pre-run value - - if ($type != "link" && $wopt_paranoid) { - - $restat = chop(@shell_exec("stat -x ".$shellpath." 2>&1")); - $post_access = null; - $post_modify = null; - $post_change = null; - - foreach (explode("\n", $restat) as $line) { - $check = substr($line, 0, 6); - if ($check == "Access") { $post_access = $line; } - if ($check == "Modify") { $post_modify = $line; } - if ($check == "Change") { $post_change = $line; } + $oops = 0; + foreach ($files as $splFileInfo) { + $path = $splFileInfo->getRealPath(); + if (!is_readable($path)) { + $oops = 1; + echo "x"; + } else { + echo "."; } - - $message = array(); - if ($pre_access != $post_access) { - $message[] = "ATIME CHANGED"; - } - if ($pre_modify != $post_modify) { - $message[] = "MTIME CHANGED"; - } - if ($pre_change != $post_change) { - $message[] = "CTIME CHANGED"; - } - - if (count($message)) { echo bashcolor(str_repeat(" ",99).implode(" ~ ", $message), "purple"); } - } + + echo "\n\n"; + + if ($oops) { + echo "Some files could not be read. Continue? (Y/n)"; + $line = trim(fgets(fopen("php://stdin","r"))); + $line = $line ?: "y"; + if($line != "y"){ + echo "Exiting!\n"; die; + } + } + + } else { + + echo bashcolor("Running as root. Some QuickLook plugins may not be available.", "red"); + echo "\n\n"; + + } +*/ + +$fixatimes = 0; +if ($wopt_paranoid) { + + echo bashcolor("\nFilesystem is writable. You can choose:\n(c) Preserve ctimes (default)\n(a) Preserve atimes\n", "purple"); + $line = trim(fgets(fopen("php://stdin","r"))) ?: "c"; + if ($line == "a") { $fixatimes = 1; } } -echo "\n\n\n"; +// Prescan +////////////////////////////////////////// -// Footer +$i = 0; +$family = array(); +$fids = array(); +foreach ($files as $splFileInfo) { + + $pathname = $splFileInfo->getPathname(); + $path = $splFileInfo->getPath(); + + $key = md5($pathname); + $pkey = md5($path); + + if (array_key_exists($key, $family)) { + echo "Duplicate key on ".$pathname."\n"; die; + } + + $family[$key] = array(); + + // Path-agnostic Unique File ID (to prevent redundant hashes and thumbs) + + if ($splFileInfo->getType() != "dir" && $splFileInfo->getType() != "link") { + + $fid = md5($splFileInfo->getSize().$splFileInfo->getMtime().$splFileInfo->getBasename()); + $dx[$fid][] = $pathname; + $fx[] = array($fid, $pathname); + $family[$key]['fid'] = $fid; + + } + + // Parents + + //$family[$key]['parents'] = getParents($zpath, $pathname); + + // Children + + if (strlen($path) > strlen($zpath)-1) { // prevent zpath root orphan + $family[$pkey]['children'][] = $key; + } + + if ($i % 5000 == 0) { + echo "\r\033[K\rPrescan: ".$pathname; + } + + $i++; + + } + +echo "\r\033[K\rPrescan: done\n"; + +// Debug record of duplicate FIDs + +$dupes = array_filter($dx, function($a) { return count($a) > 2; }); + +ob_start(); +var_dump($dupes); +$dxo = ob_get_clean(); + +if (strlen($dxo)) { + file_put_contents($bpath."/".$stamp."_dupes.txt",$dxo); + $dupecount = count($dupes,COUNT_RECURSIVE) - count($dupes); + echo "\n".bashcolor(floor(($dupecount/$i)*100)." percent of files look like duplicates","green")."\n\n"; + } + +// Write family to DB + +$message = "Writing family to DB: "; +$message .= $passed_file." files, "; +$message .= $passed_dir." dirs, "; +$message .= $nodescended." bundles, "; +$message .= $passed_link." links, "; +$message .= $ignored." ignored, "; +$message .= ($dupecount ? $dupecount : 0)." dupes"; + +echo ProgressBar::start($i,$message); + +foreach ($family as $key => $item) { + + echo ProgressBar::next(); + $stmt = $dbo->prepare("INSERT INTO family VALUES (:pid, :fid, :children)"); + $stmt->BindValue(":pid",$key); + if (@$item['fid']) { + $stmt->BindValue(":fid",$item['fid']); + } + if (@$item['children'] && is_array(@$item['children'])) { + $stmt->BindValue(":children",serialize($item['children'])); + } + $stmt->execute(); + + } + +echo ProgressBar::finish(); + +unset($dx, $dxo, $dupes); + +// stats +$stmt = "UPDATE _walkwalk SET "; +$stmt .= "passed_file=".$passed_file.", "; +$stmt .= "passed_dir=".$passed_dir.", "; +$stmt .= "passed_link=".$passed_link.", "; +$stmt .= "passed_total=".$i.", "; +$stmt .= "nodescended=".$nodescended.", "; +$stmt .= "ignored=".$ignored.", "; +$stmt .= "dupes=".($dupecount ? $dupecount : 0).", "; +$stmt .= "status='completed_in_".$seconds."'"; +$dbo->exec($stmt); + +// Thumbnails +////////////////////////////////////////// + +if ($wopt_thumbs) { + + $message = "Generating thumbnails..."; + echo ProgressBar::start(count($fx),$message); + + $tempdir = "/tmp/".$blabel."_".$stamp; + if (!is_dir($tempdir)) { mkdir($tempdir); } + + foreach ($fx as $array) { + + $fid = $array[0]; + $pathname = $array[1]; + $ext = pathinfo($pathname,PATHINFO_EXTENSION); + $tpath = $bpath."/thumbs/".substr($fid, 0, 2); + $tfile = $tpath."/".$fid.".jpg"; + + // HACK for ql-thumbnail bug + $t_skip = array("emlx"); + if (count($t_skip) && in_array($ext, $t_skip)) { + echo ProgressBar::next(1, "Skipping ".shortlabel(basename($pathname),$max_label)); + continue; + } + + // check to see if a thumb already exists + if (file_exists($tfile)) { + echo ProgressBar::next(1, "Thumb found for ".shortlabel(basename($pathname),$max_label)); + continue; + } else { + echo ProgressBar::next(1, "Generating thumb for ".shortlabel(basename($pathname),$max_label)); + } + + $shellpath = escapeshellarg($pathname); + $tempfile = $tempdir."/".$fid.".jpg"; + + /* + + // first try to make a thumb with external tools + $cmd = null; + if (in_array($ext, $t_files['vips'])) { + $cmd = $bin_tv." ".$shellpath." -o ".$tempfile."[Q=85,optimize_coding] --size=".$wopt_thumb_size; + } elseif (in_array($ext, $t_files['ffmpeg'])) { + $cmd = $bin_tf." -i ".$shellpath." -o ".$tempfile." -s ".$wopt_thumb_size." -c jpg -q 9"; + } + + if ($cmd) { shell_exec($cmd." 2>&1"); } + + */ + + // if those tools failed, try quicklook + if (!@filesize($tempfile)) { + $cmd = $bin_tq." ".$shellpath." ".$tempfile." public.jpeg-2000 ".$wopt_thumb_size." ".$wopt_thumb_size; + shell_exec($cmd." 2>&1"); + } + + // success, move thumb into the bundle + if (file_exists($tempfile) && @filesize($tempfile)) { + if (!is_dir($tpath)) { mkdir($tpath); } + rename($tempfile,$tfile); + } + + } + + echo ProgressBar::finish(); + + } + +// Hashes +////////////////////////////////////////// + +if ($wopt_hash) { + + $dbh = new PDO("sqlite:".$bpath."/md5.sqlite"); + $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + $dbh->exec("CREATE TABLE IF NOT EXISTS hash (fid TEXT, hash TEXT)"); + + if ($wopt_hash_limit) { + $message = "Generating hashes for files under".$wopt_hash_limit."GB"; + } else { + $message = "Generating hashes for all files"; + } + + echo ProgressBar::start(count($fx),$message); + + foreach ($fx as $array) { + $fid = $array[0]; + $pathname = $array[1]; + $size = filesize($pathname); + $limit = $wopt_hash_limit*1000000000; + $check = $dbh->query("SELECT EXISTS(SELECT 1 FROM hash WHERE fid='".$fid."')")->fetch()[0]; + if ($check) { + echo ProgressBar::next(1, "Hash already exists: ".shortlabel($pathname,$max_label)); + } elseif ($wopt_hash_limit && $size > $limit) { + echo ProgressBar::next(1, "Too big to hash: ".shortlabel($pathname,$max_label)." (".human_filesize($size).")"); + } else { + echo ProgressBar::next(1); + $stmt = $dbh->prepare("INSERT INTO hash VALUES (:fid, :hash)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":hash",md5_file($pathname)); + $stmt->execute(); + } + } + + echo ProgressBar::finish(); + + } + +// Files +////////////////////////////////////////// + + + +// Cleanup +////////////////////////////////////////// + +echo "\n"; $seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]); -$fbanner = "Finished in ".$seconds." seconds"; -echo str_repeat("-", strlen($fbanner))."\n".$fbanner."\n"; -echo "Files ignored: ".$ignored." / Files nodescended: ".$nodescended."\n"; +$dbo->exec("UPDATE _walkwalk SET status='completed_in_".$seconds."'"); +echo "Finished in ".$seconds." seconds\n\n"; -// Write app summary values - -$dbo->exec("UPDATE _walkwalk SET nodescended=".$nodescended.", ignored=".$ignored.", status='completed_in_".$seconds."'"); +unset($dbo, $dbh, $files, $family, $fx); ?> \ No newline at end of file diff --git a/web/rtc_patch.php b/web/rtc_patch.php new file mode 100644 index 0000000..7973ab2 --- /dev/null +++ b/web/rtc_patch.php @@ -0,0 +1,85 @@ +query("SELECT zpath FROM _walkwalk")->fetch()['zpath']; + $pathname_adjusted = str_replace($zpath."/", "", $pathname); + + $parts = explode("/", $pathname_adjusted); + $i = count($parts); + + echo "
"; echo "pathname = ".$pathname; print_r($parts);
+
+	while ($i) {
+		$search_path = $zpath."/".implode("/", array_slice($parts, 0, $i));
+		$id = $dbo->query("SELECT id FROM files WHERE (Pathname='".str_replace("'", "''", $search_path)."')")->fetch()['id'];
+		if ($i == count($parts)) {
+			$result[] = array(null, basename($search_path));
+			} else {
+			$result[] = array($id, basename($search_path));
+			}
+		$i--;
+		}
+
+	if ($pathname == $zpath) {
+		return array(array(null,basename($zpath)));
+		} else {
+		$result[] = array("1",basename($zpath));
+		return array_reverse($result);
+		}
+		
+	}
+
+////////////////////////////////
+
+// View
+
+$id = $_GET['id'];
+if (!$id) { $id = 1; }
+$db_file = "db/patch.sqlite3";
+
+$dbo = new PDO("sqlite:".$db_file);
+$dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
+
+//echo $dbo->query("SELECT COUNT(1) FROM files")->fetch()['COUNT(1)']." files
"; + +$view = $dbo->query("SELECT * FROM files WHERE (rowid=".$id.")")->fetchAll()[0]; + +print_r($view); + +echo "
"; + +$crumb = breadcrumbs($dbo, $view['Pathname']); + +foreach ($crumb as $myparts) { + if ($myparts[0] === null) { + echo $myparts[1]; + } else { + echo "".$myparts[1]." > "; + } + } + +echo "
"; + +$items = unserialize($dbo->query("SELECT children FROM patch WHERE (rowid=".$id.")")->fetch()['children']); + +foreach ($items as $id) { + $row_a = $dbo->query("SELECT * FROM files WHERE (rowid='".$id."')")->fetchAll()[0]; + $row_b = $dbo->query("SELECT * FROM mdls WHERE (rowid='".$id."')")->fetchAll()[0]; + $row_c = $dbo->query("SELECT * FROM metadata WHERE (rowid='".$id."')")->fetchAll()[0]; + if (!$row_a['Extension']) { + echo "".$row_a['Filename']." "; + } else { + echo $row_a['Filename']." "; + } + } + +//////////////////////////////// + +echo "
".round(microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"],2)." seconds


"; + +?> \ No newline at end of file