0.3.8.1

Yuba.php

@@ -1,11 +1,135 @@
 #!/usr/bin/php
 <?php
 
+error_reporting(E_ALL & ~E_NOTICE & ~E_WARNING);
+date_default_timezone_set("America/Los_Angeles");
+
 // Yuba
+//                                      //
 //////////////////////////////////////////
-$version = "0.2.1";
+$version = "0.3.8.1";
-$thumb_size = "256";
+
-$thumb_factor = $thumb_size/128;
+// Stat each file before and after reading (will fail on ctime unless volume is RO)
+if (in_array("-paranoid", $argv)) { $wopt_paranoid = 1; } else { $wopt_paranoid = 0; }
+
+// Do not descend into matching directories
+$wopt_nodescend = array("*.app", "*.sparsebundle", "*.Spotlight-V100", "*.fseventsd");
+
+// Ignore matching files and directories
+$wopt_ignore = array(".DS_Store", "*.jdk");
+
+if (in_array("-hash", $argv)) { $wopt_hash = 1; } else { $wopt_hash = 0; }
+//$wopt_hash_limit = ""; // don't hash large files nyi
+//$wopt_mediainfo = 1; // not yet implemented
+//$wopt_exiftool = 1; // not yet implemented
+//$wopt_thumb = 1; // not yet implemented
+
+$wopt_thumb_size = "256";
+$wopt_thumb_factor = $wopt_thumb_size/128;
+
+// Location of binaries used to collect metadata
+
+$bin_gfi = "/Applications/Xcode.app/Contents/Developer/usr/bin/GetFileInfo";
+$bin_mediainfo = "/opt/local/bin/mediainfo";
+$bin_exiftool = "/opt/local/bin/exiftool";
+
+// Functions
+
+function stringPrint($string) {
+	echo $string.str_repeat(" ", (10-strlen($string)));
+	}
+
+function getWoptString() {
+	global $wopt_nodescend, $wopt_ignore, $wopt_hash, $wopt_mediainfo, $wopt_exiftool, $wopt_thumb_size, $wopt_thumb_factor, $wopt_tmpdir, $wopt_paranoid;
+	return array(	array("nodescend", $wopt_nodescend), 
+					array("ignore", $wopt_ignore),
+					array("hash", $wopt_hash),
+					array("mediainfo", $wopt_mediainfo),
+					array("exiftool", $wopt_exiftool),
+					array("thumb_size", $wopt_thumb_size),
+					array("thumb_factor", $wopt_thumb_factor),
+					array("wopt_paranoid", $wopt_paranoid)
+					);
+	}
+
+class plistParser extends XMLReader {
+	public function parseString($string) { $this->XML($string); return $this->process(); }
+    private function process() {
+    	$this->read();
+    	if($this->nodeType !== XMLReader::DOC_TYPE || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 2); }
+		if(!$this->next("plist") || $this->nodeType !== XMLReader::ELEMENT || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 3); }
+		$plist = array(); while($this->read()) { if($this->nodeType == XMLReader::ELEMENT) { $plist[] = $this->parse_node(); } }
+		if(count($plist) == 1 && $plist[0]) { return $plist[0]; } else { return $plist; }
+		}
+	private function parse_node() {
+		if($this->nodeType !== XMLReader::ELEMENT) return;
+		switch($this->name) {
+			case 'data': return base64_decode($this->getNodeText()); break;
+			case 'real': return floatval($this->getNodeText()); break;
+			case 'string': return $this->getNodeText(); break;
+			case 'integer': return intval($this->getNodeText()); break;
+			case 'date': return $this->getNodeText(); break;
+			case 'true': return true; break;
+			case 'false': return false; break;
+			case 'array': return $this->parse_array(); break;
+			case 'dict': return $this->parse_dict(); break;
+			default: throw new Exception(sprintf("Not a valid plist. %s is not a valid type", $this->name), 4);
+			}			
+		}
+	private function parse_dict() {
+		$array = array(); $this->nextOfType(XMLReader::ELEMENT);
+		do { if($this->nodeType !== XMLReader::ELEMENT || $this->name !== "key") { if(!$this->next("key")) { return $array; } } $key = $this->getNodeText(); $this->nextOfType(XMLReader::ELEMENT); $array[$key] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); }
+		while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "dict")); return $array;
+		}
+	private function parse_array() {
+		$array = array(); $this->nextOfType(XMLReader::ELEMENT);
+		do { $array[] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); }
+		while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "array")); return $array;
+		}
+	private function getNodeText() { $string = $this->readString(); $this->nextOfType(XMLReader::END_ELEMENT); return $string; }
+	private function nextOfType() { $types = func_get_args(); $this->read(); while($this->nodeType && !(in_array($this->nodeType, $types))) { $this->read(); } }
+	private function isNodeOfTypeName($type, $name) { return $this->nodeType === $type && $this->name === $name; }
+	}
+
+function parseMediaInfo ($xml) {
+	$xml = simplexml_load_string($xml);
+    $data = array();
+    $data['version'] = (string) $xml['version'];
+    foreach ($xml->File->track as $track) {
+		$trackType = strtolower($track['type']);
+		$trackId = isset($track['streamid']) ? $track['streamid'] : 1;
+		$trackId = (string)$trackId;
+		$trackData = [];
+		foreach ($track as $rawKey => $rawVal) {
+			$key = strtolower($rawKey);
+			$val = (string)$rawVal;
+			if ($key == 'stream_identifier') { continue; }
+			if (!array_key_exists($key, $trackData)) {
+				$trackData[$key] = array($val);
+				} elseif (!in_array($val, $trackData[$key])) {
+				$trackData[$key][] = $val;
+				}
+            }
+		if ($trackType == 'general') {
+			$data['file']['general'] = $trackData;
+			} else {
+			$data['file'][$trackType][$trackId] = $trackData;
+			}
+		}
+	return $data;
+	}
+
+function bashcolor($str,$fgcolor="white",$bgcolor=null) {
+	static $fgcolors = array('black' => '0;30', 'dark gray' => '1;30', 'blue' => '0;34', 'light blue' => '1;34', 'green' => '0;32', 'light green' => '1;32', 'cyan' => '0;36', 'light cyan' => '1;36', 'red' => '0;31', 'light red' => '1;31', 'purple' => '0;35', 'light purple' => '1;35', 'brown' => '0;33', 'yellow' => '1;33', 'light gray' => '0;37', 'white' => '1;37', 'underline' => '4');
+	static $bgcolors = array('black' => '40', 'red' => '41', 'green' => '42', 'yellow' => '43', 'blue' => '44', 'magenta' => '45', 'cyan' => '46', 'light gray' => '47');
+	$out="";
+	if (!isset($fgcolors[$fgcolor])) { $fgcolor='white'; }
+	if (!isset($bgcolors[$bgcolor])) { $bgcolor=null; }
+	if ($fgcolor) { $out .= "\033[{$fgcolors[$fgcolor]}m"; }
+	if ($bgcolor) { $out .= "\033[{$bgcolors[$bgcolor]}m"; }
+	$out .= $str."\033[0m";
+	return $out;
+	}
 
 // Path arguments
 
@@ -22,8 +146,8 @@ if (isset($argv[2]) && is_dir($argv[2])) {
 date_default_timezone_set("America/Los_Angeles");
 $time_start = microtime(true);
 $stamp = date("Y-m-d_H-i-s", time());
-$tmpdir = "/tmp/WalkWalk_".$stamp."/";
+$wopt_tmpdir = "/tmp/WalkWalk_".$stamp."/";
-if (!is_dir($tmpdir)) { mkdir($tmpdir); }
+if (!is_dir($wopt_tmpdir)) { mkdir($wopt_tmpdir); }
 $base = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath));
 $dbfile = $dbprefix."/".$stamp."_".$base.".sqlite3";
 if (file_exists($dbfile)) { echo "File \"".$dbfile."\" already exists!"; die; }
@@ -90,62 +214,163 @@ $profile = shell_exec("system_profiler SPHardwareDataType SPStorageDataType SPTh
 $dbo = new PDO("sqlite:".$dbfile);
 $dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
 
-$dbo->exec("CREATE TABLE IF NOT EXISTS \"".$base."\" (
+$dbo->exec("CREATE TABLE files (
 				id INTEGER PRIMARY KEY, 
 				parent INTEGER, 
-				path TEXT, 
+				Pathname TEXT, 
-				ext TEXT, 
+				Path TEXT, 
-				type TEXT, 
+				Filename TEXT, 
-				size INTEGER, 
+				Extension TEXT, 
+				Type TEXT, 
+				stat TEXT, 
+				LinkTarget TEXT, 
+				RealPath TEXT, 
+				Inode INTEGER, 
+				Size INTEGER, 
+				Perms INTEGER, 
+				Owner TEXT, 
+				ATime INTEGER, 
+				MTime INTEGER, 
+				CTime INTEGER, 
+				gfi_type TEXT, 
+				gfi_attr TEXT, 
+				gfi_created TEXT, 
 				hash TEXT, 
-				perms INTEGER, 
+				tinfo TEXT
-				owner TEXT, 
+				)");
-				inode INTEGER, 
-				mtime INTEGER, 
-				atime INTEGER, 
-				ctime INTEGER, 
-				mdls TEXT, 
-				tags TEXT, 
-				mediainfo TEXT, 
-				exiftool TEXT, 
-				tinfo TEXT)");
 
-$dbo->exec("CREATE TABLE IF NOT EXISTS thumbs (
+$dbo->exec("CREATE TABLE mdls (
 				id INTEGER PRIMARY KEY, 
-				thumb BLOB)");
+				hasmeta INTEGER, 
+				DateAdded TEXT, 
+				ContentType TEXT, 
+				Creator TEXT, 
+				Kind TEXT, 
+				UserTags TEXT, 
+				FSInvisible INTEGER, 
+				PixelWidth INTEGER, 
+				PixelHeight INTEGER, 
+				spotlight TEXT
+				)");
+				
+$dbo->exec("CREATE TABLE metadata (
+				id INTEGER PRIMARY KEY, 
+				duration TEXT, 
+				mediainfo TEXT, 
+				exiftool TEXT
+				)");
+
+$dbo->exec("CREATE TABLE thumbs (
+				id INTEGER PRIMARY KEY, 
+				thumb BLOB
+				)");
 
 $dbo->exec("CREATE TABLE IF NOT EXISTS _walkwalk (
 				version TEXT, 
+				opts TEXT, 
 				host TEXT, 
 				zpath TEXT, 
 				type TEXT, 
+				nodescended INTEGER, 
+				ignored INTEGER, 
 				stats TEXT, 
 				diskutil TEXT, 
 				disks TEXT, 
-				profile TEXT)");
+				profile TEXT, 
+				status TEXT
+				)");
 
-$insert = "INSERT INTO _walkwalk VALUES (:version, :host, :zpath, :type, :stats, :diskutil, :disks, :profile)";
+$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :zpath, :type, :nodescended, :ignored, :stats, :diskutil, :disks, :profile, :status)");
-$stmt = $dbo->prepare($insert);
 $stmt->BindValue(":version",$version." (".posix_getuid().")");
+$stmt->BindValue(":opts",serialize(getWoptString()));
 $stmt->BindValue(":host",$host);
 $stmt->BindValue(":zpath",$zpath);
 $stmt->BindValue(":type",$type);
+$stmt->BindValue(":nodescended",null);
+$stmt->BindValue(":ignored",null);
 $stmt->BindValue(":stats",$dstats);
 $stmt->BindValue(":diskutil",$diskutil);
 $stmt->BindValue(":disks",$disks);
 $stmt->BindValue(":profile",$profile);
+$stmt->BindValue(":status","aborted");
 $stmt->execute();
 
 // Iterate
 
-$dir_iter = new RecursiveDirectoryIterator($zpath,
+$nodescended = 0;
+$ignored = 0;
+$files = new RecursiveIteratorIterator(
+	new RecursiveCallbackFilterIterator(
+		new RecursiveDirectoryIterator(
+      		$zpath,
 			RecursiveDirectoryIterator::SKIP_DOTS
-		);
+			),
-$iter = new RecursiveIteratorIterator($dir_iter, 
+		function ($current, $key, $iterator) use ($wopt_ignore, $wopt_nodescend) {
+			global $nodescended, $ignored;
+			$clean = true;
+			if (is_array($wopt_ignore)) {
+				foreach ($wopt_ignore as $wildcard) {
+					if (fnmatch($wildcard, $current->getFilename())) {
+						$clean = false;
+						$ignored++;
+						echo "\nSkipping: ".$current->getRealpath()."\n\n";
+						}
+					}
+				}
+			if (is_array($wopt_nodescend)) {
+				foreach ($wopt_nodescend as $wildcard) {
+					if (fnmatch($wildcard, $current->getPath())) {
+						$clean = false;
+						$nodescended++;
+						echo "\nNodescending: ".$current->getRealpath()."\n\n";
+						}
+					}
+				}
+			return $clean;
+			}
+		),
 	RecursiveIteratorIterator::SELF_FIRST,
 	RecursiveIteratorIterator::CATCH_GET_CHILD
 	);
 
+// Debug input array
+
+/*
+foreach ($files as $splFileInfo) {
+	echo "====================================================================================\n";
+	print_r($splFileInfo);
+	echo "====================================================================================\n";
+	echo "getRealPath =   ".$splFileInfo->getRealPath()."\n";
+	echo "getPathname =   ".$splFileInfo->getPathname()."\n";
+	echo "isLink =        ".$splFileInfo->isLink()."\n";
+	if ($splFileInfo->isLink()) {
+		echo "getLinkTarget = ".$splFileInfo->getLinkTarget()."\n";
+		} else {
+		echo "getLinkTarget = N/A\n";
+		}
+	echo "getType =       ".$splFileInfo->getType()." (".filetype($splFileInfo->getPathname()).")\n";
+	echo "isDir =         ".$splFileInfo->isDir()."\n";
+	echo "isFile =        ".$splFileInfo->isFile()."\n";
+	echo "-----------------------------------------------------------------------\n";
+	echo "getPath =       ".$splFileInfo->getPath()."\n";
+	echo "getFilename =   ".$splFileInfo->getFilename()."\n";
+	echo "getBasename =   ".$splFileInfo->getBasename()." (".basename($splFileInfo->getPathname()).")\n";
+	echo "getExtension =  ".$splFileInfo->getExtension()."\n";
+	echo "-----------------------------------------------------------------------\n";
+	if (!$splFileInfo->isLink()) {
+		echo "getATime =      ".$splFileInfo->getAtime()." (".fileatime($splFileInfo->getPathname()).")\n";
+		echo "getMTime =      ".$splFileInfo->getMtime()." (".filemtime($splFileInfo->getPathname()).")\n";
+		echo "getCTime =      ".$splFileInfo->getCtime()." (".filectime($splFileInfo->getPathname()).")\n";
+		echo "getInode =      ".$splFileInfo->getInode()." (".fileinode($splFileInfo->getPathname()).")\n";
+		echo "getPerms =      ".$splFileInfo->getPerms()." (".fileperms($splFileInfo->getPathname()).")\n";
+		echo "getGroup =      ".$splFileInfo->getGroup()." (".filegroup($splFileInfo->getPathname()).")\n";
+		echo "getOwner =      ".$splFileInfo->getOwner()." (".fileowner($splFileInfo->getPathname()).")\n";
+		echo "getSize =       ".$splFileInfo->getSize()." (".filesize($splFileInfo->getPathname()).")\n";
+		}
+	}
+die;
+*/
+
 // Check perms
 
 if (posix_getuid()) {
@@ -153,7 +378,7 @@ if (posix_getuid()) {
 	echo "You are not root. Checking file readability: ";
 	
 	$oops = 0;
-	foreach ($iter as $splFileInfo) {
+	foreach ($files as $splFileInfo) {
 		$path = $splFileInfo->getRealPath();
 		if (!is_readable($path)) {
 			$oops = 1;
@@ -175,7 +400,7 @@ if (posix_getuid()) {
 		}
 	}
 
-// Filetypes
+// Filetypes for special handling
 
 $m_files = array(	"mkv",
 					"ogg",
@@ -271,104 +496,226 @@ $e_files = array(	"ai",
 					"xmp",
 					"zip"	);
 
-// Insert
+// Inserts
 
-foreach ($iter as $splFileInfo) {
+foreach ($files as $splFileInfo) {
-	$path = $splFileInfo->getRealPath();
+
-	echo $path.": ";
+	$type = $splFileInfo->getType();
-	$line = array();
+	if ($type != "link") { $atime = $splFileInfo->getATime(); }
-	$info = pathinfo($path);
+	$pathname = $splFileInfo->getPathname();
-	$select = "SELECT id from \"".$base."\" WHERE (path=\"".$info['dirname']."\")";
+	$path = $splFileInfo->getPath();
-	$slct = $dbo->query($select);
+	$filename = $splFileInfo->getFilename();
-	$row = $slct->fetchObject();
+	$extension = $splFileInfo->getExtension();
-	if (@$row->id) {
+	$shellpath = escapeshellarg($pathname);
-		$line['parent'] = $row->id;
+	
+	if (!$type) { echo "\nBREAK: can't determine type of ".$pathname; die; }
+	
+	if ($type != "link") {
+		$stat = chop(@shell_exec("stat -x ".$shellpath." 2>&1"));
+		if ($wopt_paranoid && !$atime) { echo "\nBREAK: Cannot determine atime of ".$pathname; die; }
 		} else {
-		$line['parent'] = 0;
+		$stat = null;
 		}
-	$line['path'] = $path;
+
-	$line['ext'] = strtolower(@$info['extension']);
+	echo str_replace($zpath."/","",$pathname).": ";
-	$line['type'] = filetype($path);
+	$pad = 140;
-	if ($line['type'] == "dir") {
+	if (strlen($pathname) < $pad) {
-		$line['size'] = @shell_exec("du -ks \"".$path."\"")*1024;
+		echo str_repeat(" ",($pad-strlen($pathname)));
+		}
+
+	// Determine ID of parent dir by querying database
+
+	$parent = $dbo->query("SELECT id FROM files WHERE (Pathname='".str_replace("'", "''", $path)."')")->fetch()['id'];
+	stringPrint("parent");
+
+	// Gather file attributes
+	
+	$stmt = $dbo->prepare("INSERT INTO files VALUES (:id, :parent, :Pathname, :Path, :Filename, :Extension, :Type, :stat, :LinkTarget, :RealPath, :Inode, :Size, :Perms, :Owner, :ATime, :MTime, :CTime, :gfi_type, :gfi_attr, :gfi_created, :hash, :tinfo)");
+	
+	if ($type == "dir") {
+		$size = shell_exec("du -ks ".$shellpath)*1024;
+		} elseif ($type == "file") {
+		$size = $splFileInfo->getSize();
 		} else {
-		$line['size'] = filesize($path);
+		$size = null;
 		}
-	echo floor($line['size']/1024)."k; ";
+	$stmt->BindValue(":Size",@$size);
-	$line['hash'] = md5_file($path);
+	stringPrint(floor($size/1024)."k");
-	echo "hashed; ";
+	
-	$line['perms'] = fileperms($path);
+	if ($parent) {
-	$line['owner'] = fileowner($path).":".filegroup($path);
+		$stmt->BindValue(":parent",$parent);
-	$line['inode'] = fileinode($path);
+		} else {
-	$line['mtime'] = filemtime($path);
+		$stmt->BindValue(":parent",0);
-	$line['atime'] = fileatime($path);
-	$line['ctime'] = filectime($path);
-	echo "perms; ";
-	$spotlight = shell_exec("mdls \"".$path."\" 2>&1");
-	if ($spotlight != $path.": could not find ".$path.".\n") {
-		$line['mdls'] = $spotlight;
-		if (strpos($spotlight, "kMDItemUserTags")) {
-			$jlines = array_slice(explode("\n", @shell_exec("mdls -name kMDItemUserTags -raw \"".$path."\" 2>&1")), 1, -1);
-			$colors = array();
-			foreach ($jlines as $jline) {
-				$colors[] = chop($jline,",");
 		}
-			$line['tags'] = serialize($colors);
+	
+	$stmt->BindValue(":stat",@$stat);
+	
+	$stmt->BindValue(":Pathname",$pathname);
+	$stmt->BindValue(":Path",$path);
+	$stmt->BindValue(":Filename",$filename);
+	$stmt->BindValue(":Extension",$extension);
+	$stmt->BindValue(":Type",$type);
+	
+	if ($type == "link") {
+	
+		$stmt->BindValue(":LinkTarget",$splFileInfo->getLinkTarget());
+		$stmt->BindValue(":RealPath",$splFileInfo->getRealPath());
+	
+		} else {
+		
+		$stmt->BindValue(":Inode",$splFileInfo->getInode());
+		$stmt->BindValue(":Perms",$splFileInfo->getPerms());
+		$stmt->BindValue(":Owner",$splFileInfo->getOwner().":".$splFileInfo->getGroup());
+		
+		$stmt->BindValue(":ATime",$atime);
+		$stmt->BindValue(":CTime",$splFileInfo->getCTime());
+		$stmt->BindValue(":MTime",$splFileInfo->getMTime());
+	
+		}
+	
+	stringPrint("attr");
+
+	$gfiparts = explode("\n", chop(shell_exec($bin_gfi." -P ".$shellpath." 2>&1")));
+	if (is_array($gfiparts)) {
+		foreach ($gfiparts as $line) {
+			list($label, $value) = explode(": ", $line);
+			$gfi[$label] = isset($value) ? trim($value,"\"") : null;
 			}
 		}
-	echo "mdls; ";
+	
-	$thumb = $tmpdir.basename($path).".png";
+	$writegfitype = @$gfi['type'].":".@$gfi['creator'];
-	@exec("qlmanage -t -f ".$thumb_factor." -o ".$tmpdir." \"".$path."\" 2>&1");
+	if ($writegfitype == "\\0\\0\\0\\0:\\0\\0\\0\\0" || $writegfitype == ":") { $writegfitype = null; }
-	if (!file_exists($thumb) && (in_array($line['ext'], $m_files) | in_array($line['ext'], $e_files))) {
+	
-		@exec("ffmpegthumbnailer -i \"".$path."\" -o \"".$thumb."\" -s ".$thumb_size." -c png 2>&1");
+	$stmt->BindValue("gfi_type",$writegfitype);
-		echo "ffmpeg-";
+	$stmt->BindValue("gfi_attr",@$gfi['attributes']);
+	$stmt->BindValue("gfi_created",strtotime($gfi['created']));
+	
+	stringPrint("gfi");
+	
+	if ($wopt_hash && $type != "link") {
+		$stmt->BindValue(":hash",md5_file($pathname));
+		StringPrint("hash");
+		} else {
+		$stmt->BindValue(":hash",null);
+		StringPrint("(x)hash");
+		}
+
+	$thumb = $wopt_tmpdir.$filename.".png";
+	@exec("qlmanage -t -f ".$wopt_thumb_factor." -o ".$wopt_tmpdir." ".$shellpath." 2>&1");
+	stringPrint("thumb");
+	if ($size && !file_exists($thumb) && (in_array($extension, $m_files) || in_array($extension, $e_files))) {
+		@exec("ffmpegthumbnailer -i ".$shellpath." -o \"".$thumb."\" -s ".$wopt_thumb_size." -c png 2>&1");
+		stringPrint("fthumb");
+		} else {
+		stringPrint("(x)fthumb");
 		}
 	if (file_exists($thumb) && filesize($thumb)) {
-		$line['thumb'] = file_get_contents($thumb);
+		$stmt->BindValue(":tinfo",serialize(getimagesize($thumb)));
-		$line['tinfo'] = serialize(getimagesize($thumb));
 		}
-	echo "thumb; ";
+	
-	if ($line['ext'] && in_array($line['ext'], $m_files)) {
-		$line['mediainfo'] = shell_exec("mediainfo \"".$path."\" 2>&1");
-		}
-	echo "media; ";
-	if ($line['ext'] && in_array($line['ext'], $e_files)) {
-		$line['exiftool'] = shell_exec("exiftool \"".$path."\" 2>&1");
-		}	
-	echo "exif ";
-	$insert = "INSERT INTO \"".$base."\" VALUES (:id, :parent, :path, :ext, :type, :size, :hash, :perms, :owner, :inode, :mtime, :atime, :ctime, :mdls, :tags, :mediainfo, :exiftool, :tinfo)";
-	$stmt = $dbo->prepare($insert);
-	$stmt->BindValue(":parent",$line['parent']);
-	$stmt->BindValue(":path",$line['path']);
-	$stmt->BindValue(":ext",$line['ext']);
-	$stmt->BindValue(":type",$line['type']);
-	$stmt->BindValue(":size",$line['size']);
-	$stmt->BindValue(":hash",$line['hash']);
-	$stmt->BindValue(":perms",$line['perms']);
-	$stmt->BindValue(":owner",$line['owner']);
-	$stmt->BindValue(":inode",$line['inode']);
-	$stmt->BindValue(":mtime",$line['mtime']);
-	$stmt->BindValue(":atime",$line['atime']);
-	$stmt->BindValue(":ctime",$line['ctime']);
-	$stmt->BindValue(":mdls",$line['mdls']);
-	$stmt->BindValue(":tags",@$line['tags']);
-	$stmt->BindValue(":mediainfo",@$line['mediainfo']);
-	$stmt->BindValue(":exiftool",@$line['exiftool']);
-	$stmt->BindValue(":tinfo",@$line['tinfo']);
 	$stmt->execute();
-	echo "-> sql";
+	stringPrint("->files");
-	$insert = "INSERT INTO thumbs VALUES (:id, :thumb)";
+	
-	$stmt = $dbo->prepare($insert);
+	// Gather spotlight metadata
-	$stmt->BindValue(":thumb",@$line['thumb']);
+	
-	$stmt->execute();
+	$stmt = $dbo->prepare("INSERT INTO mdls VALUES (:id, :hasmeta, :DateAdded, :ContentType, :Creator, :Kind, :UserTags, :FSInvisible, :PixelWidth, :PixelHeight, :spotlight)");
-	if (@$line['thumb']) {
+	$mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1");
-		echo " (thumb)";
+	stringPrint("mdls");
+	
+	if ($mdls != $pathname.": could not find ".$pathname.".\n") {
+	
+		$stmt->BindValue(":hasmeta",1);
+		$parser = new plistParser();
+		$spotlight = $parser->parseString($mdls);
+		
+		$stmt->BindValue(":DateAdded",@$spotlight['kMDItemDateAdded']);
+		$stmt->BindValue(":ContentType",@$spotlight['kMDItemContentType']);
+		$stmt->BindValue(":Creator",@$spotlight['kMDItemCreator']);
+		$stmt->BindValue(":Kind",@$spotlight['kMDItemKind']);
+		if (isset($spotlight['kMDItemUserTags'])) {
+			$stmt->BindValue(":UserTags",serialize($spotlight['kMDItemUserTags']));
 			}
+		$stmt->BindValue(":FSInvisible",@$spotlight['kMDItemFSInvisible']);
+		$stmt->BindValue(":PixelWidth",@$spotlight['kMDItemPixelWidth']);
+		$stmt->BindValue(":PixelHeight",@$spotlight['kMDItemPixelHeight']);
+		$stmt->BindValue(":spotlight",serialize($spotlight));
+		
+		} else {
+		$stmt->BindValue(":hasmeta",0);
+		}
+	$stmt->execute();
+	stringPrint("->mdls");
+		
+	// Gather external metadata
+	
+	$stmt = $dbo->prepare("INSERT INTO metadata VALUES (:id, :duration, :mediainfo, :exiftool)");
+	
+	if ($type != "dir" && in_array($extension, $m_files)) {
+		$minfo = parseMediaInfo(shell_exec($bin_mediainfo." --Output=XML ".$shellpath." 2>&1"));
+		if ($minfo['file']['general']['duration'][0]) {
+			$stmt->BindValue(":duration",$minfo['file']['general']['duration'][0]);
+			} else {
+			$stmt->BindValue(":duration",null);
+			}
+		$stmt->BindValue(":mediainfo",serialize($minfo));
+		stringPrint("minfo");
+		} else {
+		$stmt->BindValue(":duration",null);
+		$stmt->BindValue(":mediainfo",null);
+		stringPrint("(x)info");
+		}
+	
+	if ($type != "dir" && $type != "link" && in_array($extension, $e_files)) {
+		$stmt->BindValue(":exiftool",serialize(eval("return ".`$bin_exiftool -php $shellpath`)));
+		stringPrint("etool");
+		} else {
+		$stmt->BindValue(":exiftool",null);
+		stringPrint("(x)etool");
+		}
+	$stmt->execute();
+	stringPrint("->meta");
+	
+	// Gather thumbnail
+	
+	$stmt = $dbo->prepare("INSERT INTO thumbs VALUES (:id, :thumb)");
+	if (file_exists($thumb) && filesize($thumb)) {
+		$stmt->BindValue(":thumb",file_get_contents($thumb));
+		} else {
+		$stmt->BindValue(":thumb",null);
+		}
+	$stmt->execute();
+	stringPrint("->thumb");
+	
+	// Set fileatime back to original value
+	
+	if ($type != "link" && is_writable($pathname)) {
+		@exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1");
+		stringPrint("touch");
+		}
+	
 	echo "\n";
 	
+	// Double check stat for file against pre-run value
+	
+	echo bashcolor(str_replace(array("\n"," "),array(" ",""),$stat)."\n","blue");
+	
+	if ($wopt_paranoid && $type != "link") {
+		$restat = chop(@shell_exec("stat -x ".$shellpath." 2>&1"));
+		echo bashcolor(str_replace(array("\n"," "),array(" ",""),$restat)."\n","green");
+		if ($stat != $restat) { echo "\nBREAK: stat changed on ".$pathname; die; }
 		}
 	
-$fbanner = "Finished in ".floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"])." seconds";
+	}
+
+// Footer
+
+$seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]);
+$fbanner = "Finished in ".$seconds." seconds";
 echo str_repeat("-", strlen($fbanner))."\n".$fbanner."\n";
+echo "Files ignored: ".$ignored." / Files nodescended: ".$nodescended."\n";
+
+// Write app summary values
+
+$dbo->exec("UPDATE _walkwalk SET nodescended=".$nodescended.", ignored=".$ignored.", status='completed_in_".$seconds."'");
 
 ?>

web/image.php

@@ -0,0 +1,21 @@
+<?
+
+// Yuba image 0.0.1
+
+$db = $_GET['db'];
+$id = $_GET['id'];
+
+$db_file = "db/".$db.".sqlite3";
+
+if (!file_exists($db_file)) {	
+	echo "Can't find db!";
+	die;
+	}
+
+$dbo = new PDO("sqlite:".$db_file);
+$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+
+header('Content-Type: image/png');
+echo $dbo->query("SELECT thumb FROM thumbs WHERE (id=".$id.")")->fetch()['thumb'];
+
+?>

web/rtc.php

@@ -0,0 +1,261 @@
+<?
+
+/////////////////////////////////////////////////////////////////
+// Yuba RTC Browser
+/////////////////////////////////////////////////////////////////
+
+$browser_version = "0.3.8";
+
+?>
+
+<html>
+<head>
+
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, user-scalable=no" />
+<meta name="apple-mobile-web-app-capable" content="yes" />
+
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
+<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/themes/smoothness/jquery-ui.css">
+<script src="https://ajax.googleapis.com/ajax/libs/jqueryui/1.12.1/jquery-ui.min.js"></script>
+<script src="https://raw.githubusercontent.com/bd808/php-unserialize-js/master/phpUnserialize.js"></script>
+
+<script>
+
+$(document).ready(function() {
+	$("a").contextmenu(function() {
+		window.location.href=$(this).attr('title');
+		return false;
+		})
+	})
+
+</script>
+
+<style>
+
+body { margin: 20px; font-family: helvetica; font-size: 11px; }
+*:focus { outline: none !important }
+a { color: black; }
+
+div.container { display: flex; flex-flow: row wrap; justify-content: center; }
+div.flexfill { width: 220px; height: 1px; }
+div.item { width: 180px; height: 220px; padding: 20px; }
+div.item img { margin-bottom: 20px; padding: 10px; border: 1px solid grey; }
+
+span.material { display: block; word-break:break-all; }
+span.materialsub { display: block; color: #bdbdbd; }
+i { font-style: normal; color: red; }
+
+</style>
+
+</head>
+<body>
+
+<?
+
+function array_unserialize($array) {
+	foreach ($array as $key => $string) {
+		if (substr($string, 0, 2) == "a:") {
+			$array[$key] = unserialize($string);
+			}
+		}
+	return $array;
+	}
+
+function human_filesize($bytes, $decimals = 2) {
+    $size = array('B','kB','MB','GB','TB','PB','EB','ZB','YB');
+    $factor = floor((strlen($bytes) - 1) / 3);
+    return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$size[$factor];
+	}
+
+function dumpme($mixed = null) {
+  ob_start();
+  var_dump($mixed);
+  $content = ob_get_contents();
+  ob_end_clean();
+  return $content;
+  }
+
+function breadcrumbs($dbo, $pathname) {
+	$zpath = $dbo->query("SELECT zpath FROM _walkwalk")->fetch()['zpath'];
+	$pathname_adjusted = str_replace($zpath."/", "", $pathname);
+	$parts = explode("/", $pathname_adjusted);
+	$i = count($parts);	
+	while ($i) {
+		$search_path = $zpath."/".implode("/", array_slice($parts, 0, $i));
+		$id = $dbo->query("SELECT id FROM files WHERE (Pathname='".$search_path."')")->fetch()['id'];
+		$result[] = array($id, basename($search_path));
+		$i--;
+		}
+	$home = array("0",basename($zpath));
+	if (basename($zpath) == $pathname) {
+		return array($home);
+		} else {
+		$result[] = $home;
+		return array_reverse($result);
+		}
+	}
+
+$db_dir = "db/";
+$db = $_GET['db'];
+$id = $_GET['id'];
+$view = $_GET['view'];
+$sort = $_GET['sort'];
+$db_file = $db_dir.$db.".sqlite3";
+
+if (!$view) { $view = "icon"; }
+
+// there is no db, show a list of sqlite files
+
+if (!$db) {
+
+	echo "<div class='container'>";
+	$files = glob($db_dir."*.sqlite3");
+	rsort($files);
+	foreach ($files as $file) {
+		$dbo = new PDO("sqlite:".$file);
+		$dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
+		$row = $dbo->query("SELECT * FROM _walkwalk")->fetchAll();
+		if ($row[0]['type'] == "External disk") { $icon = "icons/firewire.png"; }
+		if ($row[0]['type'] == "Startup disk") { $icon = "icons/internal.png"; }
+		if ($row[0]['type'] == "Disk image") { $icon = "icons/image.png"; }
+		if ($row[0]['type'] == "Folder") { $icon = "icons/directory.png"; }
+		echo "\n<div class='item'>";
+		echo "\n<a href='?db=".basename($file,".sqlite3")."&view=".$view."'><img src='".$icon."' title='".$row[0]['stats']."' width='128' height='128'></a>";
+		$parts = explode("_",basename($file, ".sqlite3")); 
+		echo "\n<span class='material'>".$parts[2]." (".$parts[0].")</a></span>";
+		echo "\n<span class='materialsub'>".$dbo->query("SELECT Count(*) FROM files")->fetch()['Count(*)']." Files";
+		if ($dbo->query("SELECT status FROM _walkwalk")->fetch()[0] == "aborted") {
+			echo " <i>(Aborted)</i>";
+			}
+		echo "\n</span>";
+		echo "\n</div>";
+		}
+		
+	} else {
+	
+	$dbo = new PDO("sqlite:".$db_file);
+	$dbo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
+	$zpath = $dbo->query("SELECT zpath FROM _walkwalk")->fetch()['zpath'];
+
+	// there is no id, must be initial view
+	
+	if (!$id) {
+	
+		$id = 0;
+		$type = "dir";
+		$myview['Pathname'] = basename($zpath);
+		
+		} else {
+		
+		$myview = $dbo->query("SELECT * FROM files WHERE (id=".$id.")")->fetchAll()[0];
+		$type = $myview['Type'];
+		
+		}
+
+	echo "<table width='100%'><tr><td>";
+
+	$crumb = breadcrumbs($dbo, $myview['Pathname']);
+	foreach ($crumb as $myparts) {
+		if ($crumb[count($crumb)-1] != $myparts) {
+			echo "\n<a href='finder.php?db=".$db."&id=".$myparts[0]."&view=".$view."'>".$myparts[1]."</a>";
+			echo " > ";
+			} else {
+			echo $myparts[1];
+			}
+		}
+
+	echo "</td><td width='200'>";
+
+	if ($view == "icon") {
+		echo "icon | <a href='finder.php?db=".$db."&id=".$id."&view=list'>list</a>";
+		} else {
+		echo "<a href='finder.php?db=".$db."&id=".$id."&view=icon'>icon</a> | list";
+		}
+
+	echo "</td></tr></table>";
+
+	echo "<hr>";
+
+	echo "<div class='container'>";
+	
+	// directory view
+	
+	if ($type == "dir") {
+		
+		$items = $dbo->query("SELECT * FROM files WHERE (parent=".$id.")")->fetchAll();
+		
+		if ($view == "icon") {
+		
+			foreach ($items as $item) {
+				echo "\n<div class='item'>";
+				echo "\n<a href='finder.php?db=".$db."&id=".$item['id']."&view=".$view."' title='launch.php?db=".$db."&id=".$item['id']."'>";
+				if ($item['Type'] == "dir") {
+				
+					echo "<img src='/icons/directory.png' width='128' height='128'></a>";
+					
+					} elseif (isset($item['tinfo'])) {
+					
+					list ($twidth, $theight) = unserialize($item['tinfo']);
+					echo "<img src='/image.php?db=".$db."&id=".$item['id']."' width='".($twidth/2)."' height='".($theight/2)."'>";
+					
+					} else {
+					
+					echo "<img src='/icons/null.png' width='128' height='128'>";
+					
+					}
+					
+				echo "</a>";
+				echo "\n<span class='material'>".$item['Filename'];
+				if ($item['Type'] == "dir") {
+					echo " (".count($dbo->query("SELECT * FROM files WHERE (parent=".$item['id'].")")->fetchAll())." Items)";
+					}
+				echo "\n</span>";
+				echo "\n<span class='materialsub'>".human_filesize($item['Size'])."</span>";
+				echo "\n</div>";
+				
+				}
+				
+			} else {
+			
+			echo "list";
+			
+			}
+		
+		} else {
+		
+		// file view
+		
+		echo "<pre>";
+		
+		$fdeep = $dbo->query("SELECT * FROM files WHERE (id=".$id.")")->fetchAll()[0];
+		$sdeep = $mdls = $dbo->query("SELECT * FROM mdls WHERE (id=".$id.")")->fetchAll()[0];
+		$mdeep = $meta = $dbo->query("SELECT * FROM metadata WHERE (id=".$id.")")->fetchAll()[0];
+		
+		echo "<h1>".$fdeep['Pathname']."</h1><hr>";
+		
+		if (isset($fdeep['tinfo'])) {
+			echo "<img src='/image.php?db=".$db."&id=".$id."' width='".$twidth."' height='".$theight."'><br>";
+			} else {
+			echo "<img src='/icons/null.png' width='128' height='128'><br>";
+			}
+		
+		echo dumpme(array_unserialize($fdeep));
+		echo "<h1>Spotlight</h1><hr>";
+		echo dumpme(array_unserialize($sdeep));
+		echo "<h1>Mediainfo</h1><hr>";
+		echo dumpme(array_unserialize($mdeep));
+		
+		echo "</pre>";
+		
+		}
+	
+	}
+
+echo str_repeat("<div class='flexfill'></div>", 100);
+
+?>
+
+</div>
+</body>
+</html>