831 lines
22 KiB
PHP
Executable File
831 lines
22 KiB
PHP
Executable File
#!/usr/bin/php
|
|
<?php
|
|
|
|
// Yuba
|
|
// //
|
|
//////////////////////////////////////////
|
|
$version = "0.6.0";
|
|
|
|
ini_set('memory_limit', '4096M');
|
|
date_default_timezone_set("America/Los_Angeles");
|
|
$time_start = microtime(true);
|
|
|
|
include('ProgressBar.php');
|
|
|
|
// Path & application variables
|
|
//////////////////////////////////////////
|
|
|
|
if (in_array("-nohash", $argv)) { $wopt_hash = 0; } else { $wopt_hash = 1; }
|
|
if (in_array("-nothumbs", $argv)) { $wopt_thumbs = 0; } else { $wopt_thumbs = 1; }
|
|
|
|
$zpath = realpath(@$argv[1]);
|
|
$bdest = realpath(@$argv[2]);
|
|
if (!is_dir($zpath) | !is_dir($bdest)) { echo "Usage: walk <path> <dest>"; die; }
|
|
|
|
// Check for bundle
|
|
if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); }
|
|
if (is_writable($zpath)) { $wopt_paranoid = 1; } else { $wopt_paranoid = 0; }
|
|
|
|
$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle";
|
|
if (!is_dir($bpath)) { mkdir($bpath); }
|
|
if (!is_dir($bpath."/thumbs")) { mkdir($bpath."/thumbs"); }
|
|
|
|
$wopt_hash_limit = 1; // don't hash if exceeds in gigs, 0 for unlimited
|
|
$wopt_thumb_size = "512";
|
|
|
|
// Treat these directories as files
|
|
$wopt_bundles = array( "app",
|
|
"bundle",
|
|
"sparsebundle",
|
|
"photoslibrary",
|
|
"aplibrary",
|
|
"apvault",
|
|
"abbu",
|
|
"calendar",
|
|
"framework",
|
|
"plugin",
|
|
"kext",
|
|
"rtfd"
|
|
);
|
|
|
|
foreach ($wopt_bundles as $bundle) {
|
|
$wopt_nodescend[] = "*.".$bundle;
|
|
}
|
|
|
|
// Ignore matching files and directories
|
|
$wopt_ignore = array( ".DS_Store",
|
|
".DocumentRevisions-V100",
|
|
".Spotlight-V100",
|
|
".TemporaryItems",
|
|
".apdisk",
|
|
".com.apple.timemachine.donotpresent",
|
|
".fseventsd",
|
|
".metadata-never-index",
|
|
".neofinder.abemeda.volinfo.xml"
|
|
);
|
|
|
|
$max_label = 50;
|
|
|
|
// Metadata tools
|
|
$bin_gfi = "/Applications/Xcode.app/Contents/Developer/usr/bin/GetFileInfo";
|
|
$bin_mediainfo = "/opt/local/bin/mediainfo";
|
|
$bin_exiftool = "/opt/local/bin/exiftool";
|
|
$bin_tq = "/opt/local/bin/ql-thumbnail";
|
|
$bin_tv = "/opt/local/bin/vipsthumbnail";
|
|
$bin_tf = "/usr/local/bin/ffmpegthumbnailer";
|
|
|
|
// Media extensions
|
|
//////////////////////////////////////////
|
|
|
|
$t_files['ffmpeg'] = array( "mkv",
|
|
"avi",
|
|
"mpeg",
|
|
"mpg",
|
|
"vob",
|
|
"mp4",
|
|
"m4v",
|
|
"m2v",
|
|
"m2ts",
|
|
"asf",
|
|
"wmv",
|
|
"rm",
|
|
"divx",
|
|
"fla",
|
|
"flv",
|
|
"webm" );
|
|
|
|
$t_files['vips'] = array( "jpg",
|
|
"jpeg",
|
|
"tif",
|
|
"tiff",
|
|
"gif",
|
|
"psd",
|
|
"png" );
|
|
|
|
$m_files = array( "mkv",
|
|
"ogg",
|
|
"avi",
|
|
"wav",
|
|
"mpeg",
|
|
"mpg",
|
|
"vob",
|
|
"mp4",
|
|
"m2v",
|
|
"mp3",
|
|
"asf",
|
|
"wma",
|
|
"wmv",
|
|
"qt",
|
|
"mov",
|
|
"rm",
|
|
"ifo",
|
|
"ac3",
|
|
"dts",
|
|
"aac",
|
|
"ape",
|
|
"flac",
|
|
"aiff",
|
|
"m2ts" );
|
|
|
|
$e_files = array( "ai",
|
|
"aiff",
|
|
"ape",
|
|
"asf",
|
|
"avi",
|
|
"bmp",
|
|
"divx",
|
|
"dng",
|
|
"doc",
|
|
"docx",
|
|
"eps",
|
|
"epub",
|
|
"exe",
|
|
"exif",
|
|
"fla",
|
|
"flac",
|
|
"flv",
|
|
"gif",
|
|
"icc",
|
|
"iso",
|
|
"jpg",
|
|
"jpeg",
|
|
"m2ts",
|
|
"m4a",
|
|
"m4b",
|
|
"m4v",
|
|
"mkv",
|
|
"mobi",
|
|
"azw",
|
|
"azw3",
|
|
"mov",
|
|
"qt",
|
|
"mp3",
|
|
"mp4",
|
|
"mpeg",
|
|
"mpg",
|
|
"m2v",
|
|
"nef",
|
|
"numbers",
|
|
"ogg",
|
|
"pages",
|
|
"pdf",
|
|
"pict",
|
|
"png",
|
|
"ppm",
|
|
"ppt",
|
|
"psd",
|
|
"psb",
|
|
"qif",
|
|
"raw",
|
|
"rtf",
|
|
"sr2",
|
|
"srf",
|
|
"svg",
|
|
"swf",
|
|
"tiff",
|
|
"tif",
|
|
"torrent",
|
|
"vcf",
|
|
"vob",
|
|
"wav",
|
|
"webm",
|
|
"wma",
|
|
"wmv",
|
|
"xls",
|
|
"xlsx",
|
|
"xmp",
|
|
"zip" );
|
|
|
|
foreach ($e_files as $ext) { $e_files[] = strtoupper($ext); }
|
|
foreach ($m_files as $ext) { $m_files[] = strtoupper($ext); }
|
|
foreach ($t_files['ffmpeg'] as $ext) { $t_files['ffmpeg'][] = strtoupper($ext); }
|
|
foreach ($t_files['vips'] as $ext) { $t_files['vips'][] = strtoupper($ext); }
|
|
|
|
// Functions
|
|
//////////////////////////////////////////
|
|
|
|
/*
|
|
function getParents($zpath, $pathname) {
|
|
$path = dirname($pathname);
|
|
$parts = explode("/",trim(substr($path,strlen(basename($zpath))),"/"));
|
|
foreach ($parts as $index => $part) {
|
|
$parents[] = array($part, md5($zpath."/".implode("/",array_slice($parts, 0, $index+1))));
|
|
}
|
|
return $parents;
|
|
}
|
|
*/
|
|
|
|
function shortlabel($pathname, $max) {
|
|
$basename = basename($pathname);
|
|
$suffix = "(...).".pathinfo($basename,PATHINFO_EXTENSION);
|
|
if (strlen($basename) > $max) {
|
|
return substr($basename, 0, ($max-strlen($suffix))).$suffix;
|
|
} else {
|
|
return $basename;
|
|
}
|
|
}
|
|
|
|
function human_filesize($bytes, $decimals = 2) {
|
|
$size = array('B','kB','MB','GB','TB','PB','EB','ZB','YB');
|
|
$factor = floor((strlen($bytes) - 1) / 3);
|
|
return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$size[$factor];
|
|
}
|
|
|
|
function stringPrint($string) {
|
|
echo $string.@str_repeat(" ", (10-strlen($string)));
|
|
}
|
|
|
|
function getWoptString() {
|
|
global $wopt_bundles, $wopt_ignore, $wopt_hash, $wopt_hash_limit, $wopt_mediainfo, $wopt_exiftool, $wopt_thumbs, $wopt_thumb_size, $wopt_paranoid;
|
|
return array( array("bundles", $wopt_bundles),
|
|
array("ignore", $wopt_ignore),
|
|
array("hash", $wopt_hash),
|
|
array("wopt_hash_limit", $wopt_hash_limit),
|
|
array("mediainfo", $wopt_mediainfo),
|
|
array("exiftool", $wopt_exiftool),
|
|
array("thumbs", $wopt_thumbs),
|
|
array("thumb_size", $wopt_thumb_size),
|
|
array("wopt_paranoid", $wopt_paranoid),
|
|
);
|
|
}
|
|
|
|
class plistParser extends XMLReader {
|
|
public function parseString($string) { $this->XML($string); return $this->process(); }
|
|
private function process() {
|
|
$this->read();
|
|
if($this->nodeType !== XMLReader::DOC_TYPE || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 2); }
|
|
if(!$this->next("plist") || $this->nodeType !== XMLReader::ELEMENT || $this->name !== "plist") { throw new Exception(sprintf("Error parsing plist. nodeType: %d -- Name: %s", $this->nodeType, $this->name), 3); }
|
|
$plist = array(); while($this->read()) { if($this->nodeType == XMLReader::ELEMENT) { $plist[] = $this->parse_node(); } }
|
|
if(count($plist) == 1 && $plist[0]) { return $plist[0]; } else { return $plist; }
|
|
}
|
|
private function parse_node() {
|
|
if($this->nodeType !== XMLReader::ELEMENT) return;
|
|
switch($this->name) {
|
|
case 'data': return base64_decode($this->getNodeText()); break;
|
|
case 'real': return floatval($this->getNodeText()); break;
|
|
case 'string': return $this->getNodeText(); break;
|
|
case 'integer': return intval($this->getNodeText()); break;
|
|
case 'date': return $this->getNodeText(); break;
|
|
case 'true': return true; break;
|
|
case 'false': return false; break;
|
|
case 'array': return $this->parse_array(); break;
|
|
case 'dict': return $this->parse_dict(); break;
|
|
default: throw new Exception(sprintf("Not a valid plist. %s is not a valid type", $this->name), 4);
|
|
}
|
|
}
|
|
private function parse_dict() {
|
|
$array = array(); $this->nextOfType(XMLReader::ELEMENT);
|
|
do { if($this->nodeType !== XMLReader::ELEMENT || $this->name !== "key") { if(!$this->next("key")) { return $array; } } $key = $this->getNodeText(); $this->nextOfType(XMLReader::ELEMENT); $array[$key] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); }
|
|
while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "dict")); return $array;
|
|
}
|
|
private function parse_array() {
|
|
$array = array(); $this->nextOfType(XMLReader::ELEMENT);
|
|
do { $array[] = $this->parse_node(); $this->nextOfType(XMLReader::ELEMENT, XMLReader::END_ELEMENT); }
|
|
while($this->nodeType && !$this->isNodeOfTypeName(XMLReader::END_ELEMENT, "array")); return $array;
|
|
}
|
|
private function getNodeText() { $string = $this->readString(); $this->nextOfType(XMLReader::END_ELEMENT); return $string; }
|
|
private function nextOfType() { $types = func_get_args(); $this->read(); while($this->nodeType && !(in_array($this->nodeType, $types))) { $this->read(); } }
|
|
private function isNodeOfTypeName($type, $name) { return $this->nodeType === $type && $this->name === $name; }
|
|
}
|
|
|
|
function parseMediaInfo ($xml) {
|
|
$xml = simplexml_load_string($xml);
|
|
$data = array();
|
|
$data['version'] = (string) $xml['version'];
|
|
foreach ($xml->File->track as $track) {
|
|
$trackType = strtolower($track['type']);
|
|
$trackId = isset($track['streamid']) ? $track['streamid'] : 1;
|
|
$trackId = (string)$trackId;
|
|
$trackData = [];
|
|
foreach ($track as $rawKey => $rawVal) {
|
|
$key = strtolower($rawKey);
|
|
$val = (string)$rawVal;
|
|
if ($key == 'stream_identifier') { continue; }
|
|
if (!array_key_exists($key, $trackData)) {
|
|
$trackData[$key] = array($val);
|
|
} elseif (!in_array($val, $trackData[$key])) {
|
|
$trackData[$key][] = $val;
|
|
}
|
|
}
|
|
if ($trackType == 'general') {
|
|
$data['file']['general'] = $trackData;
|
|
} else {
|
|
$data['file'][$trackType][$trackId] = $trackData;
|
|
}
|
|
}
|
|
return $data;
|
|
}
|
|
|
|
function bashcolor($str,$fgcolor="white",$bgcolor=null) {
|
|
static $fgcolors = array('black' => '0;30', 'dark gray' => '1;30', 'blue' => '0;34', 'light blue' => '1;34', 'green' => '0;32', 'light green' => '1;32', 'cyan' => '0;36', 'light cyan' => '1;36', 'red' => '0;31', 'light red' => '1;31', 'purple' => '0;35', 'light purple' => '1;35', 'brown' => '0;33', 'yellow' => '1;33', 'light gray' => '0;37', 'white' => '1;37', 'underline' => '4');
|
|
static $bgcolors = array('black' => '40', 'red' => '41', 'green' => '42', 'yellow' => '43', 'blue' => '44', 'magenta' => '45', 'cyan' => '46', 'light gray' => '47');
|
|
$out="";
|
|
if (!isset($fgcolors[$fgcolor])) { $fgcolor='white'; }
|
|
if (!isset($bgcolors[$bgcolor])) { $bgcolor=null; }
|
|
if ($fgcolor) { $out .= "\033[{$fgcolors[$fgcolor]}m"; }
|
|
if ($bgcolor) { $out .= "\033[{$bgcolors[$bgcolor]}m"; }
|
|
$out .= $str."\033[0m";
|
|
return $out;
|
|
}
|
|
|
|
// Disk info
|
|
//////////////////////////////////////////
|
|
|
|
$host = gethostname();
|
|
$disks = shell_exec("diskutil list 2>&1");
|
|
|
|
if (substr($zpath, 0, 9) != "/Volumes/") {
|
|
$zbase = "/";
|
|
} else {
|
|
$zparts = explode("/", $zpath);
|
|
$zbase = "/Volumes/".$zparts[2];
|
|
}
|
|
|
|
$diskutil = shell_exec("diskutil info ".$zbase." 2>&1");
|
|
$getstats = array( "Volume Name",
|
|
"Protocol",
|
|
"Volume UUID",
|
|
"Device Location",
|
|
"Volume Total Space",
|
|
"Volume Available Space",
|
|
"Level Type"
|
|
);
|
|
foreach ($getstats as $stat) {
|
|
preg_match("/(".$stat.":)(.*)(\n)/",$diskutil,$matches);
|
|
if (isset($matches[2])) {
|
|
if (substr($stat, -5, 5) == "Space") {
|
|
$pieces = explode(" ", trim($matches[2]));
|
|
$summary = $pieces[0]." ".$pieces[1];
|
|
$stats[$stat] = $summary;
|
|
} else {
|
|
$stats[$stat] = trim($matches[2]);
|
|
}
|
|
}
|
|
}
|
|
|
|
$dstats = serialize($stats);
|
|
if ($zpath == "/") {
|
|
$type = "Startup disk";
|
|
} elseif (strtolower($zpath) == strtolower("/Volumes/".$stats["Volume Name"])) {
|
|
if ($stats["Protocol"] == "Disk Image") {
|
|
$type = "Disk image";
|
|
} else {
|
|
$type = "External disk";
|
|
}
|
|
} else {
|
|
$type = "Folder";
|
|
}
|
|
|
|
$profile = shell_exec("system_profiler SPHardwareDataType SPStorageDataType SPThunderboltDataType SPUSBDataType 2>&1");
|
|
$qlmanage = shell_exec("qlmanage -m 2>&1");
|
|
$sysvers = shell_exec("sw_vers 2>&1");
|
|
|
|
// Database
|
|
//////////////////////////////////////////
|
|
|
|
$stamp = date("Y-m-d_H-i-s", time());
|
|
|
|
$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite");
|
|
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
|
|
|
/*
|
|
$dbo->query("PRAGMA page_size = 4096");
|
|
$dbo->query("PRAGMA cache_size = 10000");
|
|
$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
|
|
$dbo->query("PRAGMA synchronous = NORMAL");
|
|
$dbo->query("PRAGMA journal_mode = WAL");
|
|
*/
|
|
|
|
$dbo->exec("CREATE TABLE _walkwalk (
|
|
version TEXT,
|
|
opts TEXT,
|
|
host TEXT,
|
|
uid INTEGER,
|
|
zpath TEXT,
|
|
bpath TEXT,
|
|
type TEXT,
|
|
passed_file INTEGER,
|
|
passed_dir INTEGER,
|
|
passed_link INTEGER,
|
|
passed_total INTEGER,
|
|
nodescended INTEGER,
|
|
ignored INTEGER,
|
|
dupes INTEGER,
|
|
stats TEXT,
|
|
qlmanage TEXT,
|
|
sysvers TEXT,
|
|
diskutil TEXT,
|
|
disks TEXT,
|
|
profile TEXT,
|
|
status TEXT
|
|
)");
|
|
|
|
$dbo->exec("CREATE TABLE family (
|
|
pid TEXT,
|
|
fid TEXT,
|
|
children TEXT
|
|
)");
|
|
|
|
$dbo->exec("CREATE TABLE files (
|
|
pid TEXT,
|
|
fid TEXT,
|
|
Pathname TEXT,
|
|
Path TEXT,
|
|
Filename TEXT,
|
|
Extension TEXT,
|
|
Type TEXT,
|
|
items INTEGER,
|
|
newest INTEGER,
|
|
stat TEXT,
|
|
LinkTarget TEXT,
|
|
RealPath TEXT,
|
|
Inode INTEGER,
|
|
Size INTEGER,
|
|
Perms INTEGER,
|
|
Owner TEXT,
|
|
ATime INTEGER,
|
|
MTime INTEGER,
|
|
CTime INTEGER,
|
|
gfi_type TEXT,
|
|
gfi_attr TEXT,
|
|
gfi_created TEXT,
|
|
hash TEXT,
|
|
tinfo TEXT,
|
|
hasmeta INTEGER,
|
|
DateAdded TEXT,
|
|
ContentType TEXT,
|
|
Creator TEXT,
|
|
Kind TEXT,
|
|
UserTags TEXT,
|
|
FSInvisible INTEGER,
|
|
PixelWidth INTEGER,
|
|
PixelHeight INTEGER,
|
|
spotlight TEXT,
|
|
duration TEXT,
|
|
mediainfo TEXT,
|
|
exiftool TEXT
|
|
)");
|
|
|
|
$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :profile, :status)");
|
|
$stmt->BindValue(":version",$version);
|
|
$stmt->BindValue(":opts",serialize(getWoptString()));
|
|
$stmt->BindValue(":host",$host);
|
|
$stmt->BindValue(":uid",posix_getuid());
|
|
$stmt->BindValue(":zpath",$zpath);
|
|
$stmt->BindValue(":bpath",$bpath);
|
|
$stmt->BindValue(":type",$type);
|
|
$stmt->BindValue(":stats",$dstats);
|
|
$stmt->BindValue(":qlmanage",$qlmanage);
|
|
$stmt->BindValue(":sysvers",$sysvers);
|
|
$stmt->BindValue(":diskutil",$diskutil);
|
|
$stmt->BindValue(":disks",$disks);
|
|
$stmt->BindValue(":profile",$profile);
|
|
$stmt->BindValue(":status","aborted");
|
|
$stmt->execute();
|
|
|
|
// Iterator
|
|
//////////////////////////////////////////
|
|
|
|
$passed_file = $passed_dir = $passed_link = $nodescended = $ignored = 0;
|
|
$files = new RecursiveIteratorIterator(
|
|
new RecursiveCallbackFilterIterator(
|
|
new RecursiveDirectoryIterator(
|
|
// start in parent dir to include self
|
|
dirname($zpath),
|
|
RecursiveDirectoryIterator::SKIP_DOTS
|
|
),
|
|
function ($current, $key, $iterator) use ($wopt_ignore, $wopt_nodescend) {
|
|
global $nodescended, $ignored, $passed_file, $passed_dir, $passed_link, $zpath;
|
|
$clean = true;
|
|
// ensure we don't traverse zpath siblings
|
|
if ($zpath != "/" && (substr($current->getRealpath(), 0, strlen($zpath)+1) != $zpath."/") && ($current->getRealpath() != $zpath)) {
|
|
$clean = false;
|
|
}
|
|
// filenames to ignore
|
|
if (is_array($wopt_ignore)) {
|
|
foreach ($wopt_ignore as $wildcard) {
|
|
if (fnmatch($wildcard, $current->getFilename())) {
|
|
$clean = false;
|
|
$ignored++;
|
|
//echo "\nSkipping: ".$current->getRealpath()."\n\n";
|
|
}
|
|
}
|
|
}
|
|
// directories to ignore
|
|
if (is_array($wopt_nodescend)) {
|
|
foreach ($wopt_nodescend as $wildcard) {
|
|
if (fnmatch($wildcard, $current->getPath())) {
|
|
$clean = false;
|
|
$nodescended++;
|
|
//echo "\nNodescending: ".$current->getRealpath()."\n\n";
|
|
}
|
|
}
|
|
}
|
|
if ($clean) {
|
|
if ($current->getType() == "file") {
|
|
$passed_file++;
|
|
} elseif ($current->getType() == "dir") {
|
|
$passed_dir++;
|
|
} elseif ($current->getType() == "link") {
|
|
$passed_link++;
|
|
}
|
|
}
|
|
return $clean;
|
|
}
|
|
),
|
|
RecursiveIteratorIterator::SELF_FIRST,
|
|
RecursiveIteratorIterator::CATCH_GET_CHILD
|
|
);
|
|
|
|
// Banner
|
|
//////////////////////////////////////////
|
|
|
|
echo "Yuba ".$version."\n";
|
|
echo "-----------------------------------------------\n";
|
|
$banner = $zpath." -> ".$bpath;
|
|
echo $banner."\n";
|
|
echo str_repeat("-", strlen($banner))."\n";
|
|
|
|
// Permissions
|
|
//////////////////////////////////////////
|
|
|
|
/*
|
|
if (posix_getuid()) {
|
|
|
|
echo bashcolor("You are not root. Checking file readability: ", "red");
|
|
|
|
echo "\n";
|
|
|
|
$oops = 0;
|
|
foreach ($files as $splFileInfo) {
|
|
$path = $splFileInfo->getRealPath();
|
|
if (!is_readable($path)) {
|
|
$oops = 1;
|
|
echo "x";
|
|
} else {
|
|
echo ".";
|
|
}
|
|
}
|
|
|
|
echo "\n\n";
|
|
|
|
if ($oops) {
|
|
echo "Some files could not be read. Continue? (Y/n)";
|
|
$line = trim(fgets(fopen("php://stdin","r")));
|
|
$line = $line ?: "y";
|
|
if($line != "y"){
|
|
echo "Exiting!\n"; die;
|
|
}
|
|
}
|
|
|
|
} else {
|
|
|
|
echo bashcolor("Running as root. Some QuickLook plugins may not be available.", "red");
|
|
echo "\n\n";
|
|
|
|
}
|
|
*/
|
|
|
|
$fixatimes = 0;
|
|
if ($wopt_paranoid) {
|
|
|
|
echo bashcolor("\nFilesystem is writable. You can choose:\n(c) Preserve ctimes (default)\n(a) Preserve atimes\n", "purple");
|
|
$line = trim(fgets(fopen("php://stdin","r"))) ?: "c";
|
|
if ($line == "a") { $fixatimes = 1; }
|
|
|
|
}
|
|
|
|
// Prescan
|
|
//////////////////////////////////////////
|
|
|
|
$i = 0;
|
|
$family = array();
|
|
$fids = array();
|
|
foreach ($files as $splFileInfo) {
|
|
|
|
$pathname = $splFileInfo->getPathname();
|
|
$path = $splFileInfo->getPath();
|
|
|
|
$key = md5($pathname);
|
|
$pkey = md5($path);
|
|
|
|
if (array_key_exists($key, $family)) {
|
|
echo "Duplicate key on ".$pathname."\n"; die;
|
|
}
|
|
|
|
$family[$key] = array();
|
|
|
|
// Path-agnostic Unique File ID (to prevent redundant hashes and thumbs)
|
|
|
|
if ($splFileInfo->getType() != "dir" && $splFileInfo->getType() != "link") {
|
|
|
|
$fid = md5($splFileInfo->getSize().$splFileInfo->getMtime().$splFileInfo->getBasename());
|
|
$dx[$fid][] = $pathname;
|
|
$fx[] = array($fid, $pathname);
|
|
$family[$key]['fid'] = $fid;
|
|
|
|
}
|
|
|
|
// Parents
|
|
|
|
//$family[$key]['parents'] = getParents($zpath, $pathname);
|
|
|
|
// Children
|
|
|
|
if (strlen($path) > strlen($zpath)-1) { // prevent zpath root orphan
|
|
$family[$pkey]['children'][] = $key;
|
|
}
|
|
|
|
if ($i % 5000 == 0) {
|
|
echo "\r\033[K\rPrescan: ".$pathname;
|
|
}
|
|
|
|
$i++;
|
|
|
|
}
|
|
|
|
echo "\r\033[K\rPrescan: done\n";
|
|
|
|
// Debug record of duplicate FIDs
|
|
|
|
$dupes = array_filter($dx, function($a) { return count($a) > 2; });
|
|
|
|
ob_start();
|
|
var_dump($dupes);
|
|
$dxo = ob_get_clean();
|
|
|
|
if (strlen($dxo)) {
|
|
file_put_contents($bpath."/".$stamp."_dupes.txt",$dxo);
|
|
$dupecount = count($dupes,COUNT_RECURSIVE) - count($dupes);
|
|
echo "\n".bashcolor(floor(($dupecount/$i)*100)." percent of files look like duplicates","green")."\n\n";
|
|
}
|
|
|
|
// Write family to DB
|
|
|
|
$message = "Writing family to DB: ";
|
|
$message .= $passed_file." files, ";
|
|
$message .= $passed_dir." dirs, ";
|
|
$message .= $nodescended." bundles, ";
|
|
$message .= $passed_link." links, ";
|
|
$message .= $ignored." ignored, ";
|
|
$message .= ($dupecount ? $dupecount : 0)." dupes";
|
|
|
|
echo ProgressBar::start($i,$message);
|
|
|
|
foreach ($family as $key => $item) {
|
|
|
|
echo ProgressBar::next();
|
|
$stmt = $dbo->prepare("INSERT INTO family VALUES (:pid, :fid, :children)");
|
|
$stmt->BindValue(":pid",$key);
|
|
if (@$item['fid']) {
|
|
$stmt->BindValue(":fid",$item['fid']);
|
|
}
|
|
if (@$item['children'] && is_array(@$item['children'])) {
|
|
$stmt->BindValue(":children",serialize($item['children']));
|
|
}
|
|
$stmt->execute();
|
|
|
|
}
|
|
|
|
echo ProgressBar::finish();
|
|
|
|
unset($dx, $dxo, $dupes);
|
|
|
|
// stats
|
|
$stmt = "UPDATE _walkwalk SET ";
|
|
$stmt .= "passed_file=".$passed_file.", ";
|
|
$stmt .= "passed_dir=".$passed_dir.", ";
|
|
$stmt .= "passed_link=".$passed_link.", ";
|
|
$stmt .= "passed_total=".$i.", ";
|
|
$stmt .= "nodescended=".$nodescended.", ";
|
|
$stmt .= "ignored=".$ignored.", ";
|
|
$stmt .= "dupes=".($dupecount ? $dupecount : 0).", ";
|
|
$stmt .= "status='completed_in_".$seconds."'";
|
|
$dbo->exec($stmt);
|
|
|
|
// Thumbnails
|
|
//////////////////////////////////////////
|
|
|
|
if ($wopt_thumbs) {
|
|
|
|
$message = "Generating thumbnails...";
|
|
echo ProgressBar::start(count($fx),$message);
|
|
|
|
$tempdir = "/tmp/".$blabel."_".$stamp;
|
|
if (!is_dir($tempdir)) { mkdir($tempdir); }
|
|
|
|
foreach ($fx as $array) {
|
|
|
|
$fid = $array[0];
|
|
$pathname = $array[1];
|
|
$ext = pathinfo($pathname,PATHINFO_EXTENSION);
|
|
$tpath = $bpath."/thumbs/".substr($fid, 0, 2);
|
|
$tfile = $tpath."/".$fid.".jpg";
|
|
|
|
// HACK for ql-thumbnail bug
|
|
$t_skip = array("emlx");
|
|
if (count($t_skip) && in_array($ext, $t_skip)) {
|
|
echo ProgressBar::next(1, "Skipping ".shortlabel(basename($pathname),$max_label));
|
|
continue;
|
|
}
|
|
|
|
// check to see if a thumb already exists
|
|
if (file_exists($tfile)) {
|
|
echo ProgressBar::next(1, "Thumb found for ".shortlabel(basename($pathname),$max_label));
|
|
continue;
|
|
} else {
|
|
echo ProgressBar::next(1, "Generating thumb for ".shortlabel(basename($pathname),$max_label));
|
|
}
|
|
|
|
$shellpath = escapeshellarg($pathname);
|
|
$tempfile = $tempdir."/".$fid.".jpg";
|
|
|
|
/*
|
|
|
|
// first try to make a thumb with external tools
|
|
$cmd = null;
|
|
if (in_array($ext, $t_files['vips'])) {
|
|
$cmd = $bin_tv." ".$shellpath." -o ".$tempfile."[Q=85,optimize_coding] --size=".$wopt_thumb_size;
|
|
} elseif (in_array($ext, $t_files['ffmpeg'])) {
|
|
$cmd = $bin_tf." -i ".$shellpath." -o ".$tempfile." -s ".$wopt_thumb_size." -c jpg -q 9";
|
|
}
|
|
|
|
if ($cmd) { shell_exec($cmd." 2>&1"); }
|
|
|
|
*/
|
|
|
|
// if those tools failed, try quicklook
|
|
if (!@filesize($tempfile)) {
|
|
$cmd = $bin_tq." ".$shellpath." ".$tempfile." public.jpeg-2000 ".$wopt_thumb_size." ".$wopt_thumb_size;
|
|
shell_exec($cmd." 2>&1");
|
|
}
|
|
|
|
// success, move thumb into the bundle
|
|
if (file_exists($tempfile) && @filesize($tempfile)) {
|
|
if (!is_dir($tpath)) { mkdir($tpath); }
|
|
rename($tempfile,$tfile);
|
|
}
|
|
|
|
}
|
|
|
|
echo ProgressBar::finish();
|
|
|
|
}
|
|
|
|
// Hashes
|
|
//////////////////////////////////////////
|
|
|
|
if ($wopt_hash) {
|
|
|
|
$dbh = new PDO("sqlite:".$bpath."/md5.sqlite");
|
|
$dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
|
$dbh->exec("CREATE TABLE IF NOT EXISTS hash (fid TEXT, hash TEXT)");
|
|
|
|
if ($wopt_hash_limit) {
|
|
$message = "Generating hashes for files under".$wopt_hash_limit."GB";
|
|
} else {
|
|
$message = "Generating hashes for all files";
|
|
}
|
|
|
|
echo ProgressBar::start(count($fx),$message);
|
|
|
|
foreach ($fx as $array) {
|
|
$fid = $array[0];
|
|
$pathname = $array[1];
|
|
$size = filesize($pathname);
|
|
$limit = $wopt_hash_limit*1000000000;
|
|
$check = $dbh->query("SELECT EXISTS(SELECT 1 FROM hash WHERE fid='".$fid."')")->fetch()[0];
|
|
if ($check) {
|
|
echo ProgressBar::next(1, "Hash already exists: ".shortlabel($pathname,$max_label));
|
|
} elseif ($wopt_hash_limit && $size > $limit) {
|
|
echo ProgressBar::next(1, "Too big to hash: ".shortlabel($pathname,$max_label)." (".human_filesize($size).")");
|
|
} else {
|
|
echo ProgressBar::next(1);
|
|
$stmt = $dbh->prepare("INSERT INTO hash VALUES (:fid, :hash)");
|
|
$stmt->BindValue(":fid",$fid);
|
|
$stmt->BindValue(":hash",md5_file($pathname));
|
|
$stmt->execute();
|
|
}
|
|
}
|
|
|
|
echo ProgressBar::finish();
|
|
|
|
}
|
|
|
|
// Files
|
|
//////////////////////////////////////////
|
|
|
|
|
|
|
|
// Cleanup
|
|
//////////////////////////////////////////
|
|
|
|
echo "\n";
|
|
|
|
$seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]);
|
|
$dbo->exec("UPDATE _walkwalk SET status='completed_in_".$seconds."'");
|
|
echo "Finished in ".$seconds." seconds\n\n";
|
|
|
|
unset($dbo, $dbh, $files, $family, $fx);
|
|
|
|
?>
|