From cbf65062902be4e5b7d27bf8b377b669693712ee Mon Sep 17 00:00:00 2001 From: profiteroles Date: Thu, 22 Jun 2017 01:09:03 -0700 Subject: [PATCH] 0.6.1.2 --- Yuba.php | 724 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 647 insertions(+), 77 deletions(-) diff --git a/Yuba.php b/Yuba.php index 5206594..6b9b313 100755 --- a/Yuba.php +++ b/Yuba.php @@ -4,7 +4,7 @@ // Yuba // // ////////////////////////////////////////// -$version = "0.6.0.1"; +$version = "0.6.1.2"; ini_set('memory_limit', '4096M'); date_default_timezone_set("America/Los_Angeles"); @@ -17,6 +17,7 @@ include('ProgressBar.php'); if (in_array("-nohash", $argv)) { $wopt_hash = 0; } else { $wopt_hash = 1; } if (in_array("-nothumbs", $argv)) { $wopt_thumbs = 0; } else { $wopt_thumbs = 1; } +if (in_array("-nometa", $argv)) { $wopt_meta = 0; } else { $wopt_meta = 1; } $zpath = realpath(@$argv[1]); $bdest = realpath(@$argv[2]); @@ -70,7 +71,7 @@ $max_label = 50; $bin_gfi = "/Applications/Xcode.app/Contents/Developer/usr/bin/GetFileInfo"; $bin_mediainfo = "/opt/local/bin/mediainfo"; $bin_exiftool = "/opt/local/bin/exiftool"; -$bin_tq = "/opt/local/bin/ql-thumbnail"; +$bin_tq = "/opt/local/bin/ql-thumbnail-lossy"; $bin_tv = "/opt/local/bin/vipsthumbnail"; $bin_tf = "/usr/local/bin/ffmpegthumbnailer"; @@ -215,14 +216,20 @@ function getParents($zpath, $pathname) { } */ -function shortlabel($pathname, $max) { +function shortlabel($pathname, $max, $min = null) { $basename = basename($pathname); $suffix = "(...).".pathinfo($basename,PATHINFO_EXTENSION); if (strlen($basename) > $max) { - return substr($basename, 0, ($max-strlen($suffix))).$suffix; + $return = substr($basename, 0, ($max-strlen($suffix))).$suffix; } else { - return $basename; + $return = $basename; } + if (strlen($return) < $min) { + $out = $return.@str_repeat(" ", ($min-strlen($return))); + } else { + $out = $return; + } + return $out; } function human_filesize($bytes, $decimals = 2) { @@ -236,13 +243,12 @@ function stringPrint($string) { } function getWoptString() { - global $wopt_bundles, $wopt_ignore, $wopt_hash, $wopt_hash_limit, $wopt_mediainfo, $wopt_exiftool, $wopt_thumbs, $wopt_thumb_size, $wopt_paranoid; + global $wopt_bundles, $wopt_ignore, $wopt_hash, $wopt_hash_limit, $wopt_meta, $wopt_thumbs, $wopt_thumb_size, $wopt_paranoid; return array( array("bundles", $wopt_bundles), array("ignore", $wopt_ignore), array("hash", $wopt_hash), array("wopt_hash_limit", $wopt_hash_limit), - array("mediainfo", $wopt_mediainfo), - array("exiftool", $wopt_exiftool), + array("metadata", $wopt_meta), array("thumbs", $wopt_thumbs), array("thumb_size", $wopt_thumb_size), array("wopt_paranoid", $wopt_paranoid), @@ -385,7 +391,7 @@ $sysvers = shell_exec("sw_vers 2>&1"); $stamp = date("Y-m-d_H-i-s", time()); -$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite"); +$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3"); $dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); /* @@ -425,45 +431,165 @@ $dbo->exec("CREATE TABLE family ( fid TEXT, children TEXT )"); - + +/* + +Gather for each file: + +A. IDs + +pid (path ID) +fid (file ID) + +B. splFileInfo methods + +Pathname +Path +Filename +Extension +Type +Inode +Perms +Owner +ATime +CTime +MTime +LinkTarget +RealPath + +C. External methods + +stat +items +newest +gfi_type +gfi_attr +gfi_created + +D. Aggregates + +Size (splFileInfo, du) +Title (exiftool, kMDItemTitle, mediainfo) +PixelWidth (kMDItemPixelWidth, exiftool, mediainfo) +PixelHeight (kMDItemPixelHeight, exiftool, mediainfo) +Duration (kMDItemDurationSeconds, mediainfo, exiftool) +DateTimeOriginal (exiftool[DateTimeOriginal],mediainfo[EncodedDate],exiftool[CreateDate,MediaCreateDate],kMDItemContentCreationDate) +Origin (exiftool[CameraModelName,Producer,CreatorTool,WriterName,Software,Encoder],mediainfo[WritingApplication]) +GPS (exiftool[GPSPosition], kMDItemLatitude.kMDItemLongitude) +Author (exiftool[Author,Artist,Creator,By-line]) + +E. Spotlight + +spotlight (whole plist) +kMDItemDateAdded +kMDItemLastUsedDate +kMDItemUseCount +kMDItemContentModificationDate +kMDItemContentType +kMDItemCreator +kMDItemFSCreatorCode +kMDItemKind +kMDItemFSTypeCode +kMDItemUserTags +kMDItemFSInvisible +kMDItemNumberOfPages +kMDItemPageHeight +kMDItemPageWidth +kMDItemWhereFroms +kMDItemEncodingApplications + +F. Pool + +has_exif +has_mediainfo +has_hash +thumb_filename +thumb_width +thumb_height + +G. Exiftool + +ProfileDescription +BitDepth +Compression +WhiteBalance +Orientation +LensType + +H. Mediainfo + +VideoFormat +AudioFormat +Tracks +Profile +Bitrate + +*/ + $dbo->exec("CREATE TABLE files ( pid TEXT, fid TEXT, - Pathname TEXT, - Path TEXT, - Filename TEXT, - Extension TEXT, - Type TEXT, - items INTEGER, - newest INTEGER, - stat TEXT, - LinkTarget TEXT, - RealPath TEXT, - Inode INTEGER, - Size INTEGER, - Perms INTEGER, - Owner TEXT, - ATime INTEGER, - MTime INTEGER, - CTime INTEGER, - gfi_type TEXT, - gfi_attr TEXT, - gfi_created TEXT, - hash TEXT, - tinfo TEXT, - hasmeta INTEGER, - DateAdded TEXT, - ContentType TEXT, - Creator TEXT, - Kind TEXT, - UserTags TEXT, - FSInvisible INTEGER, - PixelWidth INTEGER, - PixelHeight INTEGER, + Pathname TEXT, + Path TEXT, + Filename TEXT, + Extension TEXT, + Type TEXT, + Inode INTEGER, + Perms INTEGER, + Owner TEXT, + ATime INTEGER, + CTime INTEGER, + MTime INTEGER, + LinkTarget TEXT, + RealPath TEXT, + stat TEXT, + items INTEGER, + newest INTEGER, + gfi_type TEXT, + gfi_attr TEXT, + gfi_created TEXT, + Size INTEGER, + Title TEXT, + PixelWidth INTEGER, + PixelHeight INTEGER, + Duration INTEGER, + DateTimeOriginal INTEGER, + Origin TEXT, + GPS TEXT, + Author TEXT, spotlight TEXT, - duration TEXT, - mediainfo TEXT, - exiftool TEXT + kMDItemDateAdded INTEGER, + kMDItemLastUsedDate INTEGER, + kMDItemUseCount INTEGER, + kMDItemContentModificationDate INTEGER, + kMDItemContentType TEXT, + kMDItemCreator TEXT, + kMDItemFSCreatorCode TEXT, + kMDItemKind TEXT, + kMDItemFSTypeCode TEXT, + kMDItemUserTags TEXT, + kMDItemFSInvisible INTEGER, + kMDItemNumberOfPages INTEGER, + kMDItemPageHeight INTEGER, + kMDItemPageWidth INTEGER, + kMDItemWhereFroms TEXT, + kMDItemEncodingApplications TEXT, + has_exif INTEGER, + has_mediainfo INTEGER, + has_hash INTEGER, + thumb_filename TEXT, + thumb_width INTEGER, + thumb_height INTEGER, + ProfileDescription TEXT, + BitDepth INTEGER, + Compression TEXT, + Orientation INTEGER, + LensType TEXT, + VideoFormat TEXT, + AudioFormat TEXT, + Tracks INTEGER, + Profile TEXT, + Bitrate INTEGER )"); $stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :profile, :status)"); @@ -490,37 +616,31 @@ $passed_file = $passed_dir = $passed_link = $nodescended = $ignored = 0; $files = new RecursiveIteratorIterator( new RecursiveCallbackFilterIterator( new RecursiveDirectoryIterator( - // start in parent dir to include self - dirname($zpath), + $zpath, RecursiveDirectoryIterator::SKIP_DOTS ), function ($current, $key, $iterator) use ($wopt_ignore, $wopt_nodescend) { - global $nodescended, $ignored, $passed_file, $passed_dir, $passed_link, $zpath; + global $nodescended, $ignored, $passed_file, $passed_dir, $passed_link; $clean = true; - // ensure we don't traverse zpath siblings - if ($zpath != "/" && (substr($current->getRealpath(), 0, strlen($zpath)+1) != $zpath."/") && ($current->getRealpath() != $zpath)) { - $clean = false; - } - // filenames to ignore + // identify ignore files if (is_array($wopt_ignore)) { foreach ($wopt_ignore as $wildcard) { if (fnmatch($wildcard, $current->getFilename())) { $clean = false; $ignored++; - //echo "\nSkipping: ".$current->getRealpath()."\n\n"; } } } - // directories to ignore + // identify nodescend dirs if (is_array($wopt_nodescend)) { foreach ($wopt_nodescend as $wildcard) { if (fnmatch($wildcard, $current->getPath())) { $clean = false; $nodescended++; - //echo "\nNodescending: ".$current->getRealpath()."\n\n"; } } } + //tally stats if ($clean) { if ($current->getType() == "file") { $passed_file++; @@ -549,7 +669,6 @@ echo str_repeat("-", strlen($banner))."\n"; // Permissions ////////////////////////////////////////// -/* if (posix_getuid()) { echo bashcolor("You are not root. Checking file readability: ", "red"); @@ -584,7 +703,6 @@ if (posix_getuid()) { echo "\n\n"; } -*/ $fixatimes = 0; if ($wopt_paranoid) { @@ -595,6 +713,16 @@ if ($wopt_paranoid) { } +// Pool DB +////////////////////////////////////////// + +$dbp = new PDO("sqlite:".$bpath."/pool.sqlite3"); +$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); +$dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)"); +$dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)"); +$dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)"); +$dbp->exec("CREATE TABLE IF NOT EXISTS thumbs (fid TEXT, created INTEGER, relative_path TEXT, width INTEGER, height INTEGER, tool TEXT)"); + // Prescan ////////////////////////////////////////// @@ -632,9 +760,7 @@ foreach ($files as $splFileInfo) { // Children - if (strlen($path) > strlen($zpath)-1) { // prevent zpath root orphan - $family[$pkey]['children'][] = $key; - } + $family[$pkey]['children'][] = $key; if ($i % 5000 == 0) { echo "\r\033[K\rPrescan: ".$pathname; @@ -699,8 +825,7 @@ $stmt .= "passed_link=".$passed_link.", "; $stmt .= "passed_total=".$i.", "; $stmt .= "nodescended=".$nodescended.", "; $stmt .= "ignored=".$ignored.", "; -$stmt .= "dupes=".($dupecount ? $dupecount : 0).", "; -$stmt .= "status='completed_in_".$seconds."'"; +$stmt .= "dupes=".($dupecount ? $dupecount : 0); $dbo->exec($stmt); // Thumbnails @@ -729,13 +854,20 @@ if ($wopt_thumbs) { continue; } - // check to see if a thumb already exists + // if no thumb file, then poll database if (file_exists($tfile)) { - echo ProgressBar::next(1, "Thumb found for ".shortlabel(basename($pathname),$max_label)); + echo ProgressBar::next(1, "Thumb file found for ".shortlabel(basename($pathname),$max_label)); + continue; + } elseif ($dbp->query("SELECT EXISTS(SELECT 1 FROM thumbs WHERE fid='".$fid."')")->fetch()[0]) { + echo ProgressBar::next(1, "Thumb record found for ".shortlabel(basename($pathname),$max_label)); continue; } else { echo ProgressBar::next(1, "Generating thumb for ".shortlabel(basename($pathname),$max_label)); } + + $stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":created",time()); $shellpath = escapeshellarg($pathname); $tempfile = $tempdir."/".$fid.".jpg"; @@ -743,42 +875,96 @@ if ($wopt_thumbs) { // first try to make a thumb with external tools $cmd = null; if (in_array($ext, $t_files['vips'])) { - $cmd = $bin_tv." ".$shellpath." -o ".$tempfile."[Q=85,optimize_coding] --size=".$wopt_thumb_size; + $cmd = $bin_tv." ".$shellpath." -o ".$tempfile."[Q=90,optimize_coding] --size=".$wopt_thumb_size; + $stmt->BindValue(":tool","vips"); } elseif (in_array($ext, $t_files['ffmpeg'])) { - $cmd = $bin_tf." -i ".$shellpath." -o ".$tempfile." -s ".$wopt_thumb_size." -c jpg -q 9"; + $cmd = $bin_tf." -i ".$shellpath." -o ".$tempfile." -s ".$wopt_thumb_size." -c jpg -q 8.5"; + $stmt->BindValue(":tool","ffmpeg"); } if ($cmd) { shell_exec($cmd." 2>&1"); } // if those tools failed, try quicklook if (!@filesize($tempfile)) { - $cmd = $bin_tq." ".$shellpath." ".$tempfile." public.jpeg-2000 ".$wopt_thumb_size." ".$wopt_thumb_size; + $cmd = $bin_tq." ".$shellpath." ".$tempfile." public.jpeg-2000 ".$wopt_thumb_size." ".$wopt_thumb_size." .8"; shell_exec($cmd." 2>&1"); + $stmt->BindValue(":tool","quicklook"); } // success, move thumb into the bundle if (file_exists($tempfile) && @filesize($tempfile)) { if (!is_dir($tpath)) { mkdir($tpath); } rename($tempfile,$tfile); + $stmt->BindValue(":relative_path",substr($tfile, strlen($bpath))); + list($width, $height) = getimagesize($tfile); + $stmt->BindValue(":width",$width); + $stmt->BindValue(":height",$height); } + $stmt->execute(); + } echo ProgressBar::finish(); } +// External metadata +////////////////////////////////////////// + +if ($wopt_meta) { + + $message = "Collecting external metadata..."; + echo ProgressBar::start(count($fx),$message); + + foreach ($fx as $array) { + + $fid = $array[0]; + $pathname = $array[1]; + $shellpath = escapeshellarg($pathname); + $ext = pathinfo($pathname,PATHINFO_EXTENSION); + + if (!in_array($ext, $e_files) && !in_array($ext, $m_files)) { + echo ProgressBar::next(1, "Not a media file: ".shortlabel($pathname,$max_label)); + continue; + } else { + echo ProgressBar::next(1, "Metadata: ".shortlabel($pathname,$max_label)); + } + + if (in_array($ext, $e_files)) { + $check = $dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch()[0]; + if (!$check) { + $rawexif = eval("return ".`$bin_exiftool -php $shellpath`); + $stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":tags",serialize($rawexif[0])); + $stmt->execute(); + } + } + + if (in_array($ext, $m_files)) { + $check = $dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch()[0]; + if (!$check) { + $stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)"); + $stmt->BindValue(":fid",$fid); + $stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1")))); + $stmt->execute(); + } + } + + } + + echo ProgressBar::finish(); + + } + // Hashes ////////////////////////////////////////// if ($wopt_hash) { - - $dbh = new PDO("sqlite:".$bpath."/md5.sqlite"); - $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); - $dbh->exec("CREATE TABLE IF NOT EXISTS hash (fid TEXT, hash TEXT)"); if ($wopt_hash_limit) { - $message = "Generating hashes for files under ".$wopt_hash_limit."GB"; + $message = "Generating hashes for files under".$wopt_hash_limit."GB"; } else { $message = "Generating hashes for all files"; } @@ -790,14 +976,14 @@ if ($wopt_hash) { $pathname = $array[1]; $size = filesize($pathname); $limit = $wopt_hash_limit*1000000000; - $check = $dbh->query("SELECT EXISTS(SELECT 1 FROM hash WHERE fid='".$fid."')")->fetch()[0]; + $check = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0]; if ($check) { echo ProgressBar::next(1, "Hash already exists: ".shortlabel($pathname,$max_label)); - } elseif ($wopt_hash_limit && $size > $limit) { + } elseif ($wopt_hash_limit && ($size > $limit)) { echo ProgressBar::next(1, "Too big to hash: ".shortlabel($pathname,$max_label)." (".human_filesize($size).")"); } else { - echo ProgressBar::next(1); - $stmt = $dbh->prepare("INSERT INTO hash VALUES (:fid, :hash)"); + echo ProgressBar::next(1, "Generating hash: ".shortlabel($pathname,$max_label)); + $stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)"); $stmt->BindValue(":fid",$fid); $stmt->BindValue(":hash",md5_file($pathname)); $stmt->execute(); @@ -811,7 +997,391 @@ if ($wopt_hash) { // Files ////////////////////////////////////////// +echo ProgressBar::start($i); +$j = 0; +foreach ($files as $splFileInfo) { + + // DB + + $stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Inode, :Perms, :Owner, :ATime, :CTime, :MTime, :LinkTarget, :RealPath, :stat, :items, :newest, :gfi_type, :gfi_attr, :gfi_created, :Size, :Title, :PixelWidth, :PixelHeight, :Duration, :DateTimeOriginal, :Origin, :GPS, :Author, :spotlight, :kMDItemDateAdded, :kMDItemLastUsedDate, :kMDItemUseCount, :kMDItemContentModificationDate, :kMDItemContentType, :kMDItemCreator, :kMDItemFSCreatorCode, :kMDItemKind, :kMDItemFSTypeCode, :kMDItemUserTags, :kMDItemFSInvisible, :kMDItemNumberOfPages, :kMDItemPageHeight, :kMDItemPageWidth, :kMDItemWhereFroms, :kMDItemEncodingApplications, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :ProfileDescription, :BitDepth, :Compression, :Orientation, :LensType, :VideoFormat, :AudioFormat, :Tracks, :Profile, :Bitrate)"); + + // Identify dir, file, link or bundle dir + + $type = $splFileInfo->getType(); + if ($type == "dir") { + foreach ($wopt_bundles as $bundle) { + $check = ".".$bundle; + if (substr($splFileInfo->getFilename(), -(strlen($check)), strlen($check)) == $check) { $type = "bundle"; } + } + } + + $stmt->BindValue(":Type",$type); + + // Cache atime before it gets modified + + if ($type != "link") { $atime = $splFileInfo->getATime(); } + + // Path basics + + $pathname = $splFileInfo->getPathname(); + $path = $splFileInfo->getPath(); + $filename = $splFileInfo->getFilename(); + $extension = $splFileInfo->getExtension(); + $shellpath = escapeshellarg($pathname); + + $stmt->BindValue(":Pathname",$pathname); + $stmt->BindValue(":Path",$path); + $stmt->BindValue(":Filename",$filename); + $stmt->BindValue(":Extension",$extension); + + if ($type == "link") { + + $stmt->BindValue(":LinkTarget",$splFileInfo->getLinkTarget()); + $stmt->BindValue(":RealPath",$splFileInfo->getRealPath()); + + } else { + + $stmt->BindValue(":Inode",$splFileInfo->getInode()); + $stmt->BindValue(":Perms",$splFileInfo->getPerms()); + $stmt->BindValue(":Owner",$splFileInfo->getOwner().":".$splFileInfo->getGroup()); + + $stmt->BindValue(":ATime",$atime); + $stmt->BindValue(":CTime",$splFileInfo->getCTime()); + $stmt->BindValue(":MTime",$splFileInfo->getMTime()); + + } + + stringPrint(shortlabel(basename($pathname),$max_label,$max_label+10)); + + // ------------------------------------------------ // + + // Get stat + + if ($type != "link") { + $stat = chop(@shell_exec("stat -x ".$shellpath." 2>&1")); + } else { + $stat = null; + } + $stmt->BindValue(":stat",@$stat); + + // Cache stat + + if ($type != "link" && $wopt_paranoid) { + $pre_access = null; + $pre_modify = null; + $pre_change = null; + foreach (explode("\n", $stat) as $line) { + $check = substr($line, 0, 6); + if ($check == "Access") { $pre_access = $line; } + if ($check == "Modify") { $pre_modify = $line; } + if ($check == "Change") { $pre_change = $line; } + } + } + + // Generate PID and FID + + $pid = md5($pathname); + $stmt->BindValue(":pid",$pid); + + if ($type == "file") { + $fid = md5($splFileInfo->getSize().$splFileInfo->getMtime().$splFileInfo->getBasename()); + $stmt->BindValue(":fid",$fid); + } else { + $stmt->BindValue(":fid",null); + } + + // Size + + if ($type == "dir" || $type == "bundle") { + $size = trim(shell_exec("du -ks ".$shellpath." | cut -f1"))*1024; + } elseif ($type == "file") { + $size = $splFileInfo->getSize(); + } else { + $size = null; + } + $stmt->BindValue(":Size",@$size); + stringPrint(floor($size/1024)."k"); + + // ------------------------------------------------ // + + // Items + + if ($type == "dir" || $type == "bundle" ) { + $items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\..*' \) | wc -l 2>&1"))-1; + $stmt->BindValue(":items",@$items); + stringPrint($items ? "ITEMS" : "items"); + } else { + $items = null; + stringPrint(" "); + } + + // ------------------------------------------------ // + + // Newest + + if ($type == "dir") { + $newest = @filemtime(chop(shell_exec("find ".$shellpath." -type f -not -path '*/\.*' -print0 | xargs -0 stat -f \"%m %N\" | sort -rn 2>&1 | head -1 | cut -f2- -d\" \""))); + $stmt->BindValue(":newest",@$newest); + stringPrint($newest ? "NEWEST" : "newest"); + } else { + $newest = null; + stringPrint(" "); + } + + // ------------------------------------------------ // + + // GetFileInfo + + $gfiparts = explode("\n", chop(shell_exec($bin_gfi." -P ".$shellpath." 2>&1"))); + if (is_array($gfiparts)) { + foreach ($gfiparts as $line) { + list($label, $value) = explode(": ", $line); + $gfi[$label] = isset($value) ? trim($value,"\"") : null; + } + } + + $writegfitype = @$gfi['type'].":".@$gfi['creator']; + if ($writegfitype == "\\0\\0\\0\\0:\\0\\0\\0\\0" || $writegfitype == ":") { $writegfitype = null; } + + $stmt->BindValue(":gfi_type",$writegfitype); + $stmt->BindValue(":gfi_attr",@$gfi['attributes']); + $stmt->BindValue(":gfi_created",strtotime($gfi['created'])); + + stringPrint("GFI"); + // ------------------------------------------------ // + + // Spotlight + + $mdls = null; + $mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1"); + + if ($mdls != $pathname.": could not find ".$pathname.".\n") { + $parser = new plistParser(); + $spotlight = $parser->parseString($mdls); + //$stmt->BindValue(":spotlight",serialize($spotlight)); + $stmt->BindValue(":spotlight",$mdls); + } else { + $spotlight = array(); + $stmt->BindValue(":spotlight",null); + } + + stringPrint($mdls ? "MDLS" : "mdls"); + + unset($breakout, $schema, $item, $ready); + + $breakout[] = array ("kMDItemDateAdded", "date"); + $breakout[] = array ("kMDItemLastUsedDate", "date"); + $breakout[] = array ("kMDItemUseCount", 0); + $breakout[] = array ("kMDItemContentModificationDate", "date"); + $breakout[] = array ("kMDItemContentType", 0); + $breakout[] = array ("kMDItemCreator", 0); + $breakout[] = array ("kMDItemFSCreatorCode", 0); + $breakout[] = array ("kMDItemKind", 0); + $breakout[] = array ("kMDItemFSTypeCode", 0); + $breakout[] = array ("kMDItemUserTags", "array"); + $breakout[] = array ("kMDItemFSInvisible", 0); + $breakout[] = array ("kMDItemNumberOfPages", 0); + $breakout[] = array ("kMDItemPageHeight", 0); + $breakout[] = array ("kMDItemPageWidth", 0); + $breakout[] = array ("kMDItemWhereFroms", "array"); + $breakout[] = array ("kMDItemEncodingApplications", "array"); + + foreach ($breakout as $schema) { + if (!isset($spotlight[$schema[0]])) { + $stmt->BindValue(":".$schema[0],null); + continue; + } + if ($schema[1] === "date") { + $ready = strtotime($spotlight[$schema[0]]); + } elseif ($schema[1] === "array") { + $ready = serialize($spotlight[$schema[0]]); + } else { + $ready = $spotlight[$schema[0]]; + } + $stmt->BindValue(":".$schema[0],$ready); + } + + unset($breakout); + + // ------------------------------------------------ // + + // Pool + + if ($type == "file") { + + unset($fetch_exif, $fetch_media, $fetch_hash, $fetch_thumb, $yes_exif, $yes_media, $yes_hash); + + $fetch_exif = @unserialize($dbp->query("SELECT tags FROM exiftool WHERE fid='".$fid."'")->fetch()[0]); + is_array($fetch_exif) ? $yes_exif = 1 : $yes_exif = 0; + $stmt->BindValue(":has_exif",$yes_exif); + stringPrint($yes_exif ? "EXIF" : "exif"); + + $fetch_media = @unserialize($dbp->query("SELECT info FROM mediainfo WHERE fid='".$fid."'")->fetch()[0]); + is_array($fetch_media) ? $yes_media = 1 : $yes_media = 0; + $stmt->BindValue(":has_mediainfo",$yes_media); + stringPrint($yes_media ? "MEDIA" : "media"); + + $yes_hash = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0]; + $stmt->BindValue(":has_hash",$yes_hash); + stringPrint($yes_hash ? "HASH" : "hash"); + + $fetch_thumb = $dbp->query("SELECT * FROM thumbs WHERE fid='".$fid."'")->fetch(); + if (@$fetch_thumb['relative_path']) { + $stmt->BindValue(":thumb_filename",$fetch_thumb['relative_path']); + $stmt->BindValue(":thumb_width",$fetch_thumb['width']); + $stmt->BindValue(":thumb_height",$fetch_thumb['height']); + stringPrint("THUMB"); + } else { + $stmt->BindValue(":thumb_filename",null); + stringPrint("thumb"); + } + + $breakout[] = "ProfileDescription"; + $breakout[] = "BitDepth_BitsPerSample"; + $breakout[] = "Compression"; + $breakout[] = "Aperture,LightSource,WhiteBalance"; + $breakout[] = "Orientation"; + $breakout[] = "LensType,FocalLength"; + + $breakout['profile'] = "ProfileDescription"; + $breakout['bits'] = "BitDepth_BitsPerSample"; + $breakout['compression'] = "Compression"; + $breakout[] = "Aperture,LightSource,WhiteBalance"; + $breakout[] = "Orientation"; + $breakout[] = "LensType,FocalLength"; + +/* + +function parseConditionalItem($data, $item) { + if (strpos($item, "_")) { + $list = explode("_", $item); + foreach ($list as $piece) { + + // left off here + + } + } + + +function parseItem($data, $item) { + if (strpos($item, ",")) { + $list = explode(",", $item); + foreach ($list as $piece) { + $cleared[] = parseConditionalItem($piece); + } + } else { + $cleared[] = parseConditionalItem($item); + } + foreach ($cleared as $check) { + if (isset($data[$check])) { + $ready[] = $data[$check]; + } + } + if (@count($ready) > 1) { + return implode(", ",$ready); + } elseif (@count($ready) == 1) { + return $ready[0]; + } else { + return null; + } + } + +// left off here + + foreach ($breakout as $item) { + unset($ready); + if (strpos($item, ",")) { + $list = explode(",", $item) { + foreach ($list as $multi) { + if (isset($fetch_exif[$multi])) { + $ready[] = $fetch_exif[$multi]; + } + } + } elseif (strpos($item, "_")) { + $list = explode("_", $item) { + foreach ($list as $multi) { + if (isset($ready) { + continue; + } + if (isset($fetch_exif[$multi])) { + $ready[] = $fetch_exif[$multi]; + } + + +*/ + + unset($breakout); + + print_r($fetch_exif); + print_r($fetch_media); + + } else { + + stringPrint(" "); + stringPrint(" "); + stringPrint(" "); + stringPrint(" "); + + } + + // Write to DB + + $stmt->execute(); + stringPrint("DB"); + + // Set fileatime back to original value + + if ($type != "link" && is_writable($pathname) && $fixatimes) { + @exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1"); + stringPrint("touch"); + } + + echo "\n"; + + // Double check stat for file against pre-run value + + if ($type != "link" && $wopt_paranoid) { + + $restat = chop(@shell_exec("stat -x ".$shellpath." 2>&1")); + $post_access = null; + $post_modify = null; + $post_change = null; + + foreach (explode("\n", $restat) as $line) { + $check = substr($line, 0, 6); + if ($check == "Access") { $post_access = $line; } + if ($check == "Modify") { $post_modify = $line; } + if ($check == "Change") { $post_change = $line; } + } + + $message = array(); + if ($pre_access != $post_access) { + $message[] = "ATIME"; + } + if ($pre_modify != $post_modify) { + $message[] = "MTIME"; + } + if ($pre_change != $post_change) { + $message[] = "CTIME"; + } + + if (count($message)) { stringPrint("Change: ".implode(", ", $message)); } + + } + + $update = ProgressBar::next(1, substr($pathname,0,80)); + if (floor($j/100) == ($j/100)) { + echo "\n\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n"; + echo $update; + echo "\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n"; + } + + $j++; + + } + +echo ProgressBar::finish(); // Cleanup ////////////////////////////////////////// @@ -822,6 +1392,6 @@ $seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]); $dbo->exec("UPDATE _walkwalk SET status='completed_in_".$seconds."'"); echo "Finished in ".$seconds." seconds\n\n"; -unset($dbo, $dbh, $files, $family, $fx); +unset($dbo, $dbp, $files, $family, $fx); ?> \ No newline at end of file