p/Yuba
1
0
Fork 0
Code Issues 1 Pull Requests Releases 6 Wiki Activity

0.7.12.1

Browse Source
This commit is contained in:
profiteroles
2019-05-23 04:48:00 -07:00
parent e84c693464
commit 4f090db179
8 changed files with 122 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -4,7 +4,7 @@
**Yuba** generates a web-browsable SQLite database from an HFS+ filesystem. Its client application gathers forensic-quality data about a locally attached disk, properly interpreting bundles, reading Spotlight data, Finder flags, labels, and other contextual information. It can generate hashes, thumbnails, and gather 3rd party metadata with exiftool and mediainfo. Yuba's filesystem catalogues are comprehensive, lightweight, optimized for massive (1 million+) trees, and reflect incremental changes to contents and metadata. A server-side PHP script is provided, which allows familiar, Finder-style browsing of a catalogue. **Yuba** generates a web-browsable SQLite database from an HFS+ filesystem. Its client application gathers forensic-quality data about a locally attached disk, properly interpreting bundles, reading Spotlight data, Finder flags, labels, and other contextual information. It can generate hashes, thumbnails, and gather 3rd party metadata with exiftool and mediainfo. Yuba's filesystem catalogues are comprehensive, lightweight, optimized for massive (1 million+) trees, and reflect incremental changes to contents and metadata. A server-side PHP script is provided, which allows familiar, Finder-style browsing of a catalogue.
* **⇩ [Download Yuba 0.7.12](http://www.profiteroles.org/downloads/Yuba_0.7.12.zip)** * **⇩ [Download Yuba 0.7.12.1](http://www.profiteroles.org/downloads/Yuba_0.7.12.1.zip)**
## Features ## Features

128
Yuba.php
View File

@@ -11,7 +11,14 @@ date_default_timezone_set("America/Los_Angeles");
// Includes & Prefs // Includes & Prefs
////////////////////////////////////////// //////////////////////////////////////////
$p = unserialize(file_get_contents(__DIR__."/prefs.php")); $prefs_file = "/Users/".get_current_user()."/Library/Preferences/yuba_prefs.php";
if (!file_exists($prefs_file)) {
if (!copy(__DIR__."/prefs.php",$prefs_file)) {
echo "Error creating preferences file";
die;
}
}
$p = unserialize(file_get_contents($prefs_file));
$p['phpbin'] = "/usr/bin/php"; $p['phpbin'] = "/usr/bin/php";
@@ -75,6 +82,8 @@ $bin_mediainfo = __DIR__."/bin/mediainfo";
$bin_exiftool = __DIR__."/bin/exiftool"; $bin_exiftool = __DIR__."/bin/exiftool";
$bin_ffmpeg = __DIR__."/bin/ffmpeg"; $bin_ffmpeg = __DIR__."/bin/ffmpeg";
$bin_qlthumb = __DIR__."/bin/ql-thumbnail"; $bin_qlthumb = __DIR__."/bin/ql-thumbnail";
$bin_qltool = __DIR__."/bin/qltool";
$bin_vips = "vipsthumbnail";
// Logfile // Logfile
$messages_log_file = $bpath."/".$stamp."_messages.log"; $messages_log_file = $bpath."/".$stamp."_messages.log";
@@ -275,6 +284,7 @@ $dbo->exec("CREATE TABLE files (
thumb_filename TEXT, thumb_filename TEXT,
thumb_width INTEGER, thumb_width INTEGER,
thumb_height INTEGER, thumb_height INTEGER,
thumb_tool TEXT,
contents_filename TEXT contents_filename TEXT
)"); )");
@@ -537,63 +547,102 @@ if ($p['thumbs']) {
$ext = pathinfo($pathname,PATHINFO_EXTENSION); $ext = pathinfo($pathname,PATHINFO_EXTENSION);
$tpath = $bpath."/thumbs/".substr($fid, 0, 2); $tpath = $bpath."/thumbs/".substr($fid, 0, 2);
if (!is_dir($tpath)) { mkdir($tpath); } if (!is_dir($tpath)) { mkdir($tpath); }
$tfile = $tpath."/".$fid.".jpg"; $tfile = $tpath."/".$fid."_".$p['thumb_size'].".jpg";
$tpfile = $tpath."/".$fid."_".$p['thumb_size'].".png";
if (count($p['t_skip']) && in_array($ext, $p['t_skip'])) { if (count($p['t_skip']) && in_array($ext, $p['t_skip'])) {
echo ProgressBar::next("Skipping ".shortlabel($pathname)); echo ProgressBar::next("Skipping ".shortlabel($pathname));
continue; continue;
} }
// if no thumb file, then poll database // check for rebuild mode
if (file_exists($tfile)) { if ($p['thumbs'] == 2) {
echo ProgressBar::next("Thumb file found for ".shortlabel($pathname)); $update_rowid = $dbp->query("SELECT rowid FROM thumbs WHERE fid='".$fid."'")->fetch()[0];
continue; } else {
} elseif ($dbp->query("SELECT EXISTS(SELECT 1 FROM thumbs WHERE fid='".$fid."')")->fetch()[0]) { if (is_file($tfile) | is_file($tpfile)) {
echo ProgressBar::next("Thumb record found for ".shortlabel($pathname)); // faster to check for a file first
continue; echo ProgressBar::next("Thumb file found for ".shortlabel($pathname));
continue;
} elseif ($dbp->query("SELECT EXISTS(SELECT 1 FROM thumbs WHERE fid='".$fid."')")->fetch()[0]) {
// if no thumb file, then poll database
echo ProgressBar::next("Thumb record found for ".shortlabel($pathname));
continue;
} else {
$update_rowid = false;
}
}
if ($update_rowid) {
echo ProgressBar::next("Updating thumb for ".shortlabel($pathname));
$stmt = $dbp->prepare("UPDATE thumbs SET fid = :fid, created = :created, relative_path = :relative_path, width = :width, height = :height, tool = :tool WHERE rowid = ".$update_rowid);
} else { } else {
echo ProgressBar::next("Generating thumb for ".shortlabel($pathname)); echo ProgressBar::next("Generating thumb for ".shortlabel($pathname));
$stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)");
} }
$stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)");
$stmt->BindValue(":fid",$fid);
$stmt->BindValue(":created",time());
$shellpath = escapeshellarg($pathname); $shellpath = escapeshellarg($pathname);
$tmp_path = escapeshellarg("/tmp/".basename($pathname).".png"); // qlmanage workaround
$fmt['sips'] = $tfile;
$fmt['ffmpeg'] = $tfile;
$fmt['ql-thumbnail'] = $tfile;
$fmt['qltool'] = $tpfile;
$fmt['qlmanage'] = $tpfile;
$fmt['vips'] = $tfile;
$cmd['sips'] = "sips -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile;
$cmd['ffmpeg'] = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile;
$cmd['ql-thumbnail'] = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8";
$cmd['qltool'] = $bin_qltool." di ".$shellpath." ".($p['thumb_size']/2)." ".($p['thumb_size']/2)." | base64 --decode > ".$tpfile;
$cmd['qlmanage'] = "qlmanage -ti -f ".floor($p['thumb_size']/128)." -o /tmp/ ".$shellpath."; mv ".$tmp_path." ".$tpfile;
$cmd['vips'] = $bin_vips." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size'];
// first try to make a thumb with external tools
$cmd = null;
if (in_array($ext, $p['t_files']['sips'])) { if (in_array($ext, $p['t_files']['sips'])) {
//$cmd = $bin_tv." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size']; $external_tool = "sips";
$cmd = "sips -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile;
$stmt->BindValue(":tool","sips");
} elseif (in_array($ext, $p['t_files']['ffmpeg'])) { } elseif (in_array($ext, $p['t_files']['ffmpeg'])) {
//$cmd = $bin_tf." -i ".$shellpath." -o ".$tfile." -s ".$p['thumb_size']." -c jpg -q 8.5"; $external_tool = "ffmpeg";
$cmd = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath." | cut -d'.' -f1) / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile; } else {
$stmt->BindValue(":tool","ffmpeg"); $external_tool = null;
} }
if ($cmd) { shell_exec($cmd." 2>&1"); } // using ql-tool or qlmanage may generate blank media icons
switch ($p['thumb_priority']) {
// if those tools failed, try quicklook case 0:
if (!@filesize($tfile)) { // external tool priority
//$cmd = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg-2000 ".$p['thumb_size']." ".$p['thumb_size']." .8"; $priority = array($external_tool,"ql-thumbnail");
$cmd = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8"; break;
shell_exec($cmd." 2>&1"); case 1:
$stmt->BindValue(":tool","quicklook"); // ql-thumbnail priority
$priority = array("ql-thumbnail",$external_tool,"qltool","qlmanage");
break;
case 2:
// qltool priority
$priority = array("qltool","qlmanage","ql-thumbnail",$external_tool);
break;
case 3:
// qlmanage priority
$priority = array("qlmanage","qltool","ql-thumbnail",$external_tool);
break;
} }
// success, move thumb into the bundle foreach ($priority as $tool) {
// ignore generic music icon thumbs (7133) if (@$cmd[$tool]) {
if (file_exists($tfile) && @filesize($tfile) && @filesize($tfile) != 7133) { shell_exec($cmd[$tool]." 2>&1");
$stmt->BindValue(":relative_path",substr($tfile, strlen($bpath))); msg($cmd[$tool]);
list($width, $height) = getimagesize($tfile); if (is_file($fmt[$tool]) && @filesize($fmt[$tool])) {
$stmt->BindValue(":width",$width); $stmt->BindValue(":fid",$fid);
$stmt->BindValue(":height",$height); $stmt->BindValue(":created",time());
$stmt->BindValue(":tool",$tool);
$stmt->BindValue(":relative_path",substr($fmt[$tool], strlen($bpath)));
list($width, $height) = getimagesize($fmt[$tool]);
$stmt->BindValue(":width",$width);
$stmt->BindValue(":height",$height);
$stmt->execute();
break;
}
}
} }
$stmt->execute();
} }
echo ProgressBar::finish(); echo ProgressBar::finish();
@@ -860,7 +909,7 @@ foreach ($files as $splFileInfo) {
// DB // DB
$stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Size, :Inode, :Perms, :Owner, :ATime, :MTime, :CTime, :LinkTarget, :RealPath, :stat, :items, :newest, :fkind, :gfi_type, :gfi_attr, :gfi_created, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :contents_filename)"); $stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Size, :Inode, :Perms, :Owner, :ATime, :MTime, :CTime, :LinkTarget, :RealPath, :stat, :items, :newest, :fkind, :gfi_type, :gfi_attr, :gfi_created, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :thumb_tool, :contents_filename)");
// Identify dir, file, link or bundle dir // Identify dir, file, link or bundle dir
@@ -1008,6 +1057,7 @@ foreach ($files as $splFileInfo) {
$stmt->BindValue(":thumb_filename",$fetch_thumb['relative_path']); $stmt->BindValue(":thumb_filename",$fetch_thumb['relative_path']);
$stmt->BindValue(":thumb_width",$fetch_thumb['width']); $stmt->BindValue(":thumb_width",$fetch_thumb['width']);
$stmt->BindValue(":thumb_height",$fetch_thumb['height']); $stmt->BindValue(":thumb_height",$fetch_thumb['height']);
$stmt->BindValue(":thumb_tool",$fetch_thumb['tool']);
} else { } else {
$stmt->BindValue(":thumb_filename",null); $stmt->BindValue(":thumb_filename",null);
} }

9
YubaPrefs.php
View File

@@ -10,7 +10,8 @@ require (__DIR__."/functions.pashua.php");
// Read Prefs // Read Prefs
$p = unserialize(file_get_contents(__DIR__."/prefs.php")); $prefs_file = "/Users/".get_current_user()."/Library/Preferences/yuba_prefs.php";
$p = unserialize(file_get_contents($prefs_file));
if(!$p['bdest']) { if(!$p['bdest']) {
$p['bdest'] = "/Users/".get_current_user()."/Documents/Yuba/"; $p['bdest'] = "/Users/".get_current_user()."/Documents/Yuba/";
if (!is_dir($p['bdest'])) { if (!mkdir($p['bdest'])) { echo "Error creating destination directory"; } } if (!is_dir($p['bdest'])) { if (!mkdir($p['bdest'])) { echo "Error creating destination directory"; } }
@@ -19,7 +20,8 @@ if(!$p['bdest']) {
// Load strings // Load strings
$strings[] = array("Do nothing","Reveal result in Finder","Upload result with rsync"); $strings[] = array("Do nothing","Reveal result in Finder","Upload result with rsync");
$strings[] = array("Bypass","Generate","Rebuild (tk)"); $strings[] = array("Bypass","Generate","Rebuild");
$strings[] = array("external","ql-thumbnail","qltool","qlmanage");
$result = Pashua::showDialog(makeWindowString($p, $strings)); $result = Pashua::showDialog(makeWindowString($p, $strings));
@@ -34,6 +36,7 @@ if (@$result['cb']) {
$result['postflight'] = array_search($result['postflight'],$strings[0]); $result['postflight'] = array_search($result['postflight'],$strings[0]);
$result['thumbs'] = array_search($result['thumbs'],$strings[1]); $result['thumbs'] = array_search($result['thumbs'],$strings[1]);
$result['thumb_priority'] = array_search($result['thumb_priority'],$strings[2]);
// If the user didn't specify a destpath, set to default // If the user didn't specify a destpath, set to default
@@ -47,7 +50,7 @@ $result['destpath'] = str_replace("Desktop/Desktop","Desktop",$result['destpath'
// Write Prefs // Write Prefs
file_put_contents("prefs.php",serialize($result)); file_put_contents($prefs_file,serialize($result));
echo "1"; echo "1";
?> ?>

BIN
bin/qltool Executable file
View File

Binary file not shown.

23
functions.pashua.php
View File

@@ -119,14 +119,33 @@ function makeWindowString($p, $strings) {
thumbs.option = ".$strings[1][1]." thumbs.option = ".$strings[1][1]."
thumbs.option = ".$strings[1][2]." thumbs.option = ".$strings[1][2]."
thumbs.default = ".$strings[1][$p['thumbs']]." thumbs.default = ".$strings[1][$p['thumbs']]."
thumbs.width = 160 thumbs.width = 140
thumbs.type = popup
thumbs.label = Thumbnails
thumbs.option = ".$strings[1][0]."
thumbs.option = ".$strings[1][1]."
thumbs.option = ".$strings[1][2]."
thumbs.default = ".$strings[1][$p['thumbs']]."
thumbs.width = 120
thumb_priority.type = popup
thumb_priority.label = Priority
thumb_priority.option = ".$strings[2][0]."
thumb_priority.option = ".$strings[2][1]."
thumb_priority.option = ".$strings[2][2]."
thumb_priority.option = ".$strings[2][3]."
thumb_priority.default = ".$strings[2][$p['thumb_priority']]."
thumb_priority.width = 120
thumb_priority.x = 150
thumb_priority.y = 162
thumb_size.type = textfield thumb_size.type = textfield
thumb_size.default = ".$p['thumb_size']." thumb_size.default = ".$p['thumb_size']."
thumb_size.label = Size thumb_size.label = Size
thumb_size.placeholder = pixels thumb_size.placeholder = pixels
thumb_size.width = 60 thumb_size.width = 60
thumb_size.x = 200 thumb_size.x = 300
thumb_size.y = 165 thumb_size.y = 165
hash.type = checkbox hash.type = checkbox

2
prefs.php
View File

@@ -1 +1 @@
a:15:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"fixatimes";i:0;s:11:"verify_stat";i:0;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;} a:16:{s:5:"bdest";s:0:"";s:10:"rsync_dest";s:0:"";s:10:"postflight";i:1;s:11:"readability";i:0;s:9:"fixatimes";i:0;s:11:"verify_stat";i:0;s:4:"meta";i:1;s:6:"thumbs";i:1;s:10:"thumb_size";i:512;s:14:"thumb_priority";i:0;s:4:"hash";i:1;s:10:"hash_limit";i:1;s:8:"contents";i:1;s:14:"contents_limit";i:50;s:9:"spotlight";i:1;s:7:"profile";i:1;}

1
utils/generate_default_prefs
View File

@@ -12,6 +12,7 @@ $p['verify_stat'] = 0;
$p['meta'] = 1; $p['meta'] = 1;
$p['thumbs'] = 1; $p['thumbs'] = 1;
$p['thumb_size'] = 512; $p['thumb_size'] = 512;
$p['thumb_priority'] = 0;
$p['hash'] = 1; $p['hash'] = 1;
$p['hash_limit'] = 1; $p['hash_limit'] = 1;
$p['contents'] = 1; $p['contents'] = 1;

2
version.txt
View File

@@ -1 +1 @@
0.7.12 0.7.12.1