p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.7.2

Browse Source
This commit is contained in:
profiteroles
2018-09-01 21:30:02 -07:00
parent ff19eeab6c
commit 547eb3f4fa
5 changed files with 194 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
Yuba.php
View File

@@ -3,7 +3,7 @@
// Yuba
// //
//////////////////////////////////////////
$version = "0.6.9";
$version = "0.7.2";
ini_set('memory_limit', '4096M');
date_default_timezone_set("America/Los_Angeles");
@@ -29,7 +29,7 @@ if (!is_dir($zpath) | !is_dir($bdest)) { echo "Filepath error"; die; }
// Check for bundle
if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); }
if (is_writable($zpath)) { $p['paranoid'] = 1; } else { $p['paranoid'] = 0; }
if (is_writable($zpath)) { echo "Warning: source is writeable\n"; }
$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle";
if (!is_dir($bpath)) { mkdir($bpath); }
@@ -73,6 +73,13 @@ $bin_exiftool = __DIR__."/bin/exiftool";
$bin_ffmpeg = __DIR__."/bin/ffmpeg";
$bin_qlthumb = __DIR__."/bin/ql-thumbnail";
// Logfile
error_reporting(E_ALL);
ini_set("display_errors", TRUE);
ini_set("log_errors", TRUE);
ini_set("error_log", $bpath."/php.log");
// Banner
//////////////////////////////////////////
@@ -86,6 +93,13 @@ echo "Gathering system info...\n";
$host = gethostname();
$disks = shell_exec("diskutil list 2>&1");
$df = shell_exec("df 2>&1");
$df_volume = shell_exec("df ".$zpath." | tail -n 1 | rev | cut -d' ' -f1 | rev");
$df_device = shell_exec("df ".$zpath." | tail -n 1 | cut -d' ' -f1");
$mdutil = shell_exec("mdutil -s ".$df_volume);
if (strpos($mdutil,"disabled")) {
echo "Warning: spotlight indexing is disabled\n";
}
if (substr($zpath, 0, 9) != "/Volumes/") {
$zbase = "/";
@@ -175,6 +189,10 @@ $dbo->exec("CREATE TABLE _walkwalk (
sysvers TEXT,
diskutil TEXT,
disks TEXT,
df TEXT,
df_device TEXT,
df_volume TEXT,
mdutil TEXT,
profile TEXT,
status TEXT
)");
@@ -185,100 +203,6 @@ $dbo->exec("CREATE TABLE family (
children TEXT
)");
/*
Gather for each file:
A. IDs
pid (path ID)
fid (file ID)
B. splFileInfo methods
Pathname
Path
Filename
Extension
Type
Inode
Perms
Owner
ATime
CTime
MTime
LinkTarget
RealPath
C. External methods
stat
items
newest
gfi_type
gfi_attr
gfi_created
D. Aggregates
Size (splFileInfo, du)
Title (exiftool, kMDItemTitle, mediainfo)
PixelWidth (kMDItemPixelWidth, exiftool, mediainfo)
PixelHeight (kMDItemPixelHeight, exiftool, mediainfo)
Duration (kMDItemDurationSeconds, mediainfo, exiftool)
DateTimeOriginal (exiftool[DateTimeOriginal],mediainfo[EncodedDate],exiftool[CreateDate,MediaCreateDate],kMDItemContentCreationDate)
Origin (exiftool[CameraModelName,Producer,CreatorTool,WriterName,Software,Encoder],mediainfo[WritingApplication])
GPS (exiftool[GPSPosition], kMDItemLatitude.kMDItemLongitude)
Author (exiftool[Author,Artist,Creator,By-line])
E. Spotlight
spotlight (whole plist)
kMDItemDateAdded
kMDItemLastUsedDate
kMDItemUseCount
kMDItemContentModificationDate
kMDItemContentType
kMDItemCreator
kMDItemFSCreatorCode
kMDItemKind
kMDItemFSTypeCode
kMDItemUserTags
kMDItemFSInvisible
kMDItemNumberOfPages
kMDItemPageHeight
kMDItemPageWidth
kMDItemWhereFroms
kMDItemEncodingApplications
F. Pool
has_exif
has_mediainfo
has_hash
thumb_filename
thumb_width
thumb_height
G. Exiftool
ProfileDescription
BitDepth
Compression
WhiteBalance
Orientation
LensType
H. Mediainfo
VideoFormat
AudioFormat
Tracks
Profile
Bitrate
*/
$dbo->exec("CREATE TABLE files (
pid TEXT,
fid TEXT,
@@ -345,7 +269,7 @@ $dbo->exec("CREATE TABLE files (
Bitrate INTEGER
)");
$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :profile, :status)");
$stmt = $dbo->prepare("INSERT INTO _walkwalk VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)");
$stmt->BindValue(":version",$version);
$stmt->BindValue(":opts",serialize($p));
$stmt->BindValue(":host",$host);
@@ -358,6 +282,10 @@ $stmt->BindValue(":qlmanage",$qlmanage);
$stmt->BindValue(":sysvers",$sysvers);
$stmt->BindValue(":diskutil",$diskutil);
$stmt->BindValue(":disks",$disks);
$stmt->BindValue(":df",$df);
$stmt->BindValue(":df_device",$df_device);
$stmt->BindValue(":df_volume",$df_volume);
$stmt->BindValue(":mdutil",$mdutil);
$stmt->BindValue(":profile",$profile);
$stmt->BindValue(":status","aborted");
$stmt->execute();
@@ -418,53 +346,45 @@ $files = new RecursiveIteratorIterator(
foreach ($files as $null) { }
$first_run = 0;
// Permissions
// Permissions & Stat
//////////////////////////////////////////
if (posix_getuid()) {
echo ProgressBar::start($passed_total,"Stat");
$i = 0;
$noread = array();
foreach ($files as $splFileInfo) {
$shellpath = escapeshellarg($splFileInfo->getPathname());
$realpath = $splFileInfo->getRealPath();
if ($splFileInfo->getType() != "link") {
$stx[$i] = array( $splFileInfo->getATime(),
$splFileInfo->getMTime(),
$splFileInfo->getCTime() );
}
$sty[$i] = chop(shell_exec("stat -x ".$shellpath." 2>&1"));
echo ProgressBar::start($passed_total,"File permissions");
echo "You are not root. Checking file readability";
$oops = 0;
foreach ($files as $splFileInfo) {
$path = $splFileInfo->getRealPath();
if ($path && !is_readable($path)) {
$oops = 1;
echo "x";
} else {
echo ".";
}
echo ProgressBar::next();
//echo "\t".$path."\n";
if ($realpath && !is_readable($realpath)) {
$noread[] = $realpath;
}
echo ProgressBar::finish();
echo "\n";
if ($oops) {
echo "Some files could not be read. Stopping.";
echo ProgressBar::next();
$i++;
}
if (count($noread)) {
foreach ($noread as $file) {
echo "Current user (".posix_getuid().") does not have read access to ".$file."\n";
}
if ($p['readability']) {
echo "Exiting...";
die;
}
} else {
echo "Running as root. Some QuickLook plugins may not be available.";
echo "\n";
}
if ($p['paranoid']) {
echo "Filesystem is writable. ";
if ($p['fixatimes']) {
echo "Preserving atimes.";
} else {
echo "Preserving ctimes.";
}
echo "\n";
}
echo ProgressBar::finish();
// Pool DB
//////////////////////////////////////////
@@ -475,6 +395,7 @@ $dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS thumbs (fid TEXT, created INTEGER, relative_path TEXT, width INTEGER, height INTEGER, tool TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS contents (fid TEXT, created INTEGER, relative_path TEXT)");
// Prescan
//////////////////////////////////////////
@@ -577,6 +498,15 @@ $stmt .= "ignored=".$ignored.", ";
$stmt .= "dupes=".($dupecount ? $dupecount : 0);
$dbo->exec($stmt);
// Contents
//////////////////////////////////////////
if ($p['contents']) {
echo "DO CONTENTS HERE\n";
}
// Thumbnails
//////////////////////////////////////////
@@ -742,7 +672,9 @@ if ($p['hash']) {
// Files
//////////////////////////////////////////
echo ProgressBar::start($passed_total,"Processing files");
$j = 0;
echo ProgressBar::start($passed_total,"Skimming");
foreach ($files as $splFileInfo) {
@@ -764,10 +696,6 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":Type",$type);
// Cache atime before it gets modified
if ($type != "link") { $atime = $splFileInfo->getATime(); }
// Path basics
$pathname = $splFileInfo->getPathname();
@@ -781,6 +709,8 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":Filename",$filename);
$stmt->BindValue(":Extension",$extension);
$stmt->BindValue(":stat",$sty[$j]);
if ($type == "link") {
$stmt->BindValue(":LinkTarget",$splFileInfo->getLinkTarget());
@@ -792,9 +722,9 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":Perms",$splFileInfo->getPerms());
$stmt->BindValue(":Owner",$splFileInfo->getOwner().":".$splFileInfo->getGroup());
$stmt->BindValue(":ATime",$atime);
$stmt->BindValue(":CTime",$splFileInfo->getCTime());
$stmt->BindValue(":MTime",$splFileInfo->getMTime());
$stmt->BindValue(":ATime",$stx[$j][0]);
$stmt->BindValue(":MTime",$stx[$j][1]);
$stmt->BindValue(":CTime",$stx[$j][2]);
}
@@ -802,29 +732,6 @@ foreach ($files as $splFileInfo) {
// ------------------------------------------------ //
// Get stat
if ($type != "link") {
$stat = chop(@shell_exec("stat -x ".$shellpath." 2>&1"));
} else {
$stat = null;
}
$stmt->BindValue(":stat",@$stat);
// Cache stat
if ($type != "link" && $p['paranoid']) {
$pre_access = null;
$pre_modify = null;
$pre_change = null;
foreach (explode("\n", $stat) as $line) {
$check = substr($line, 0, 6);
if ($check == "Access") { $pre_access = $line; }
if ($check == "Modify") { $pre_modify = $line; }
if ($check == "Change") { $pre_change = $line; }
}
}
// Generate PID and FID
$pid = md5($pathname);
@@ -895,6 +802,7 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":gfi_created",strtotime($gfi['created']));
stringPrint("GFI");
// ------------------------------------------------ //
// Spotlight
@@ -1067,25 +975,30 @@ function parseItem($data, $item) {
// Write to DB
$stmt->execute();
stringPrint("DB");
stringPrint("->DB");
// Set fileatime back to original value
if ($type != "link" && is_writable($pathname) && $p['fixatimes']) {
@exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1");
stringPrint("touch");
}
//if ($type != "link" && is_writable($pathname) && $p['fixatimes']) {
// exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1");
// stringPrint("touch");
// }
// Double check stat for file against pre-run value
if ($type != "link" && $p['paranoid']) {
if ($type != "link") {
$restat = chop(@shell_exec("stat -x ".$shellpath." 2>&1"));
$post_access = null;
$post_modify = null;
$post_change = null;
$pre_access = $pre_modify = $pre_change = null;
$post_access = $post_modify = $post_change = null;
foreach (explode("\n", $sty[$j]) as $line) {
$check = substr($line, 0, 6);
if ($check == "Access") { $pre_access = $line; }
if ($check == "Modify") { $pre_modify = $line; }
if ($check == "Change") { $pre_change = $line; }
}
foreach (explode("\n", $restat) as $line) {
foreach (explode("\n", chop(shell_exec("stat -x ".$shellpath." 2>&1"))) as $line) {
$check = substr($line, 0, 6);
if ($check == "Access") { $post_access = $line; }
if ($check == "Modify") { $post_modify = $line; }
@@ -1109,6 +1022,7 @@ function parseItem($data, $item) {
echo "\n";
echo ProgressBar::next();
$j++;
}