#!/usr/bin/perl # by Polsy, 9/2010 # Thread URL, if not already fetched into a file $threadURL = ""; # Name the the game here - used as the main heading on the index page # and the page title of updates $gameName = "?"; # Your SA name, for the 'Collected By' line $collector = "Your SA Name Here"; # These following settings should be set to 1 to enable them, # or to 0 to disable them # Multiple videos on the same line are backups of the same video $multiBackups = 1; # You're unlikely to need to change the following settings # Optional parameters specific to wget # - Retry downloading files up to a maximum of 3 times $wgetParms = "--tries=2"; # List of sites that contain videos (so any hrefs referring to these # sites are most likely videos) @videoSites = (['Dailymotion', 'dailymotion.com' ], ['YouTube', 'youtube.com' ], ['Google', 'video.google.[^/]+'], ['FileFront', 'filefront.com' ], ['Viddler', 'viddler.com' ], ['Vimeo', 'vimeo.com' ], ['blip.tv', 'blip.tv' ]); # Location of CSS file $cssURL = "/LetsPlay/myStyle.css"; # Cookies for downloading threads from the forums: # - Auto-detect by default (currently Windows-only) if($^O eq "MSWin32") { $isWin32 = 1; ($uid, $pass) = &detectWinCookies; } elsif($^O eq "darwin") { $isMacOS = 1; ($uid, $pass) = &detectMacCookies; # Set wget executable if it isn't already if(-f 'wget' && ! -x 'wget') { system("/bin/chmod u+x wget"); } # Add the current directory to the path so wget works if it's there $ENV{'PATH'} = $ENV{'PATH'} . ':.'; } # - or manual settings: if(! $uid) { # username ('bbuserid' cookie) $uid = "123456"; # password ('bbpassword' cookie) $pass = "65517c0bb4f3b05648502d1917fb2c6a"; } $scriptStart = time; print "Script started at ", scalar(localtime($scriptStart)), "\n\n"; # If no thread URL, and there isn't exactly one argument passed # try to default to showthread.txt If that doesn't exist, fail if(! $threadURL) { if($#ARGV != 0) { if(-f "showthread.txt") { $ARGV[0] = "showthread.txt"; } else { print "Syntax: $0 filename\n"; exit 1; } } } else { &getThread($threadURL); $ARGV[0] = "autoFetchedThread.txt"; } open(IN, "<$ARGV[0]") || die "Couldn't open file $ARGV[0]: $!\n"; $upd = 0; $postBody = ""; $author = "?"; $firstDate = "?"; $threadName = "?"; $threadURL = "?"; $errors = ""; $vidCols = 1; # Make subdirectory for images mkdir "Images" || die $!; # Go through the file until you hit a post, useful thing or the file ends while() { # Found a post! if(//) { if($upd == 1) { # shouldn't be able to get here if we already # got an update, but just in case last; } &getPost; } elsif($author eq "?" && /

(.+)

#; } elsif($firstDate eq "?" && //) { # First post date $junk = ; # blank line $junk = ; # post number $junk = ; # thread/user numbers $_ = ; # actual date info ($firstDate) = m#(\w+ \d+, \d+) \d+:\d+#; # Got the post's date - we're done last; } elsif(//) { # Thread name ($threadName) = m#^\s+<title>(.+) - The Something Awful Forums#; } elsif(/Search thread: /) { # Thread ID (generate URL) ($threadID) = m#name="threadid" value="(\d+)"#; $threadURL = "http://forums.somethingawful.com/showthread.php?threadid=$threadID"; } } close(IN); # Delete temporary file if we downloaded the thread ourselves if($threadURL) { unlink 'autoFetchedThread.txt'; } # Get the current date to use as the Date Added use POSIX; $addedDate = POSIX::strftime("%b %d, %Y", localtime(time)); # Create the index open(IDX, ">index.html") || die "Couldn't create index: $!\n"; print IDX < $gameName - Index

Let's Play $gameName

Author: $author
Collected By: $collector
Original Thread: $threadName
First Update: $firstDate
Last Update: ?
Date Added: $addedDate

Statistics

HEADER1 print IDX "Number of Videos: ", scalar(grep(!/^b/, @videoURLs)), "
\n"; print IDX <

Introduction

$postBody MAIN if($#videoURLs > -1) { $vidSize = 0; print IDX <

Videos

VIDEOH if($multiBackups) { for($i=0; $i<$vidCols;$i++) { print IDX qq##; } } print IDX "\n"; $tblContent = ""; for($i=0; $i<=$#videoURLs; $i++) { $URL = $videoURLs[$i]; $title = $videoTitles[$i]; # Columns for video mirrors if($multiBackups && $vidCols > 1) { ($t1, $t2) = ($title =~ /^([^`]+)`(.+)$/); if(! $t1) { $t1 = $title; $t2 = $title; } $tblContent .= qq#

$t1	$t2\n#; while($videoURLs[$i+1] =~ /^b/) { $i++; $URL = $videoURLs[$i]; $title = $videoTitles[$i]; $URL =~ s/^b//; $tblContent .= qq#	$title\n#; } } else { $tblContent .= qq#
$title\n#; } $tblContent .= "\n"; } $tblContent .= "

\n\n"; } # Dump table to file print IDX $tblContent; # Remove wget error file (if it was generated, either here or earlier) unlink 'wgeterr.txt'; print IDX <

Back to Let's Play Index FOOTER close(IDX); if($isWin32) { $winConsole->Title("Finished - $gameName"); } if($vidSize) { $txt = sprintf "Total video size: %2.2f MB\n", ($vidSize / 1048576); print "\n$txt"; open(V, ">stats.txt"); print V $txt; close(V); } else { unlink 'stats.txt'; } if($errors) { print "\nErrors:\n"; print $errors; open(V, ">>stats.txt"); print V "\nErrors:\n"; print V $errors; close(V); } $scriptEnd = time; print "\nScript ended at ", scalar(localtime($scriptEnd)), "\n"; $elapsed = $scriptEnd - $scriptStart; print "Elapsed time: "; if($elapsed < 60) { print "$elapsed seconds\n"; } else { $elapsedM = int($elapsed / 60); $elapsedS = $elapsed % 60; printf "$elapsedM minute%s, $elapsedS second%s\n", ($elapsedM == 1 ? "":"s"), ($elapsedS == 1 ? "":"s"); } print "\nPress enter to finish\n"; $junk = ; # Subroutines # Extract an entire post from a thread list, save the images sub getPost { $upd++; $img = 0; undef %images; while() { # Ignore these next if //; next if //; # End of post marker last if //; # Fix the line ending to match the downloader's platform # (to make editing the output files easier) tr/\r\n//d; $_ .= "\n"; if(m#]+>##g; $lineTitle =~ s#!img#]+>([^<]+)#; if(! $title) { # Can happen with image-only video links ($URL) = m#]+>.+#; $title = "(no title)"; } # Video? $isVideo = 0; for $site (@videoSites) { if($URL =~ /$$site[1]/) { $isVideo = 1; $siteName = $$site[0]; } } if($isVideo) { # Avoid duplicates $gotURL = 0; for($i=0; $i<=$#videoURLs; $i++) { if($videoURLs[$i] eq $URL || $videoURLs[$i] eq "b$URL") { $gotURL = 1; } } if(! $gotURL) { if(@newURLs) { $URL = "b$URL"; } if($multiBackups) { $title = $siteName; } else { $title = "$lineTitle | $siteName"; } push(@newURLs, $URL); push(@newTitles, $title); } } # Stop this video from being processed again s# $vidCols) { $vidCols = scalar(@newURLs); } } } # Catch images (and take a copy of the post) - first post only! if($upd == 1) { # loop in case there's multiple images on one line, try repeatedly while(m#) { $errors .= $_; } close(ERR); } else { print "ok\n"; } $smilies{$imgURL} = $newImgLink; } else { # No, so make a new filename for it $img++; ($newImgLink) = ($imgURL =~ m#/([^/]+)$#); if($newImgLink !~ /\..{1,3}$/) { $newImgLink .= '.'; } # Change spaces and magic characters to _ $newImgLink =~ s/ /_/g; $newImgLink =~ s/%[\dA-F]{2}/_/g; # Add the image number to it $newImgLink = "Images/$img-$newImgLink"; # download it print "Fetching image $img ($imgURL)..."; system(qq#wget $wgetExtra$wgetParms -nv -o wgeterr.txt -O "$newImgLink" "$imgURL"#); if($? != 0) { print "errored\n"; $errors .= "* wget failure getting $imgURL for image $img\n"; open(ERR, ") { $errors .= $_; } close(ERR); } else { print "ok\n"; } # and store the old/new image links in case they're used again $images{$imgURL} = "$newImgLink"; } # Adjust the post for the new image location s/\Q$origImgURL\E/$newImgLink/; } $postBody .= $_; } } # Fix definitely typo filter $postBody =~ s/\[NOTE: I AM TOO STUPID TO SPELL THE WORD "DEFINITELY" CORRECTLY\]/definitely/g; # Switch
for
$postBody =~ s#
#
#g; } # Download original thread from SA forums sub getThread { ($threadURL) = $_[0]; print "* Fetching thread from $threadURL...\n"; if($uid) { $cookies = qq#--header "Cookie: bbuserid=$uid; bbpassword=$pass"#; } system(qq#wget $wgetParms -nv -O autoFetchedThread.txt $cookies "$threadURL"#); } # Automatically detect browser cookies (Windows) sub detectWinCookies { print "Auto-detecting cookies...\n"; # Check Firefox first # (Application Data\Mozilla\Firefox\Profiles\myProfile\cookies.txt) $cPath = $ENV{'APPDATA'} . "/Mozilla/Firefox"; if(! -d $cPath) { $noFF = "Firefox not installed"; } elsif(! open(PROINI, "<$cPath/profiles.ini")) { $noFF = "Couldn't read Firefox profiles file"; } else { while() { if(/^Path=/) { ($profDir) = /^Path=(.+)$/; last; } } close(PROINI); $cPath .= "/$profDir"; if(! -d $cPath) { $noFF = "Couldn't find a Firefox profile"; } elsif(! open(COOK, "$cPath/cookies.txt")) { $noFF = "Couldn't open Firefox cookie file"; } else { while() { if(/^forums.somethingawful.com/) { ($key, $value) = /^\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+(\S+)/; if($key eq "bbuserid") { $uid = $value; } if($key eq "bbpassword") { $pass = $value; } } } close(COOK); if(! $uid) { $noFF = "No SomethingAwful cookies found in Firefox cookies"; } else { print " Found in Firefox cookies\n"; } } } if($noFF) { print " Firefox check failed: $noFF\n"; # Internet Explorer # Can't just 'use' here because it upsets other OSs BEGIN { if($^O eq "MSWin32") { require Win32API::Registry; require Win32::Console; Win32API::Registry->import(qw(HKEY_CURRENT_USER KEY_READ RegCloseKey RegOpenKeyEx RegQueryValueEx)); $winConsole = new Win32::Console(); } } if(! RegOpenKeyEx(HKEY_CURRENT_USER, 'Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders', 0, KEY_READ, $shFKey)) { $noIE = "Couldn't open Shell Folders in registry"; } elsif(! RegQueryValueEx($shFKey, "Cookies", [], [], $cValue, [])) { $noIE = "Couldn't open IE Cookies registry key"; } elsif(! opendir(COOKDIR, $cValue)) { $noIE = "Couldn't open IE Cookies folder"; } else { @saCookies = grep(/\@forums.somethingawful/, readdir(COOKDIR)); closedir(COOKDIR); foreach $cFile (@saCookies) { if(open(COOK, "<$cValue/$cFile")) { while() { chomp($key = $_); chomp($value = ); $site = ; for($i=0;$i<6;$i++) { $junk = ; } if($key eq "bbuserid") { $uid = $value; } if($key eq "bbpassword") { $pass = $value; } } close(COOK); } # Stop checking cookie files if we got a result last if($uid); } RegCloseKey($shFKey); if(! $uid) { $noIE = "No SomethingAwful cookies found in IE cookies"; } else { print " Found in IE cookies\n"; } } } if($noIE) { print " Internet Explorer check failed: $noIE\n"; use Fcntl 'O_RDONLY', 'SEEK_CUR'; $MSB = 0x80; $cPath = $ENV{'APPDATA'} . "/Opera/Opera/profile"; if(! -d $cPath) { $noOp = "Opera not installed?\n"; } else { ($uid, $pass) = &doOpera("$cPath/cookies4.dat"); if(! $uid) { $noOp = "No SomethingAwful cookies found in Opera cookies"; } else { print " Found in Opera cookies\n"; } } } if($noOp) { print " Opera check failed: $noOp\n\n"; return ("", ""); } else { return ($uid, $pass); } } # Automatically detect browser cookies (Mac OS X) sub detectMacCookies { print "Auto-detecting cookies...\n"; # Check Firefox first # $HOME/Library/Application Support/Firefox/Profiles/myProfile/cookies.txt) $cPath = $ENV{'HOME'} . "/Library/Application Support/Firefox"; if(! -d $cPath) { $noFF = "Firefox not installed"; } elsif(! open(PROINI, "<$cPath/profiles.ini")) { $noFF = "Couldn't read Firefox profiles file"; } else { while() { if(/^Path=/) { ($profDir) = /^Path=(.+)$/; last; } } close(PROINI); $cPath .= "/$profDir"; if(! -d $cPath) { $noFF = "Couldn't find a Firefox profile"; } elsif(! open(COOK, "$cPath/cookies.txt")) { $noFF = "Couldn't open Firefox cookie file"; } else { while() { if(/^forums.somethingawful.com/) { ($key, $value) = /^\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+(\S+)/; if($key eq "bbuserid") { $uid = $value; } if($key eq "bbpassword") { $pass = $value; } } } close(COOK); if(! $uid) { $noFF = "No SomethingAwful cookies found in Firefox cookies"; } else { print " Found in Firefox cookies\n"; } } } if($noFF) { print " Firefox check failed: $noFF\n"; # Safari if(!open(COOK, $ENV{'HOME'} . "/Library/Cookies/Cookies.plist")) { $noSafari = "Couldn't open Safari cookies file"; } else { while() { next unless //; $dom = ""; $name = ""; $value = ""; while() { last if m##; ($key) = /([^<]+); ($val) = />([^<]+)32bit numbers # It doesn't matter because it's *currently* only used for timestamps return $v2; } else { print "Unhandled byte size $bytes\n"; #&end; } } sub processDomain { # Domain Record my $domName; $domPartLen = &getNum($lenLen); do { $recPart = &getNum($tagLen); $domPartLen -= $tagLen; if($recPart == 0x1E) { # Domain name $nameLen = &getNum($lenLen); $domPartLen -= $lenLen; sysread(C, $domName, $nameLen); #print "domain $domName\n"; $domPartLen -= $nameLen; $domArray[$domParts] = $domName; $domParts++; if($domArray[0] eq "com" && $domArray[1] eq "somethingawful" && $domArray[2] =~ /^forums?$/) { $sa = 1; } else { $sa = 0; } } else { #print "Unhandled domain part $recPart\n"; # 0x1F, 0x21, 0x25 which nobody seems to use } } while($domPartLen); do { $next = &getNum(1); if($next == 5 + $MSB) { # Useless empty path component } elsif($next == 4 + $MSB) { # End of domain } else { # Path or Cookie (or.......?) if($next == 1) { &processDomain; $next = -1; } elsif($next == 2) { &processPath; $next = -1; } elsif($next == 3) { &processCookie; $next = -1; } else { print "Unhandled component type $type\n"; } } } while($next != 4 + $MSB); #print "done domain $domName!\n"; $domParts--; } sub processPath { $pathLen = &getNum($lenLen); do { $pathPart = &getNum($tagLen); $pathLen -= $tagLen; if($pathPart != 0x1D) { print "Unhandled path component $pathPart\n"; } # Path string $pathStrLen = &getNum($lenLen); $pathLen -= $lenLen; sysread(C, $pathStr, $pathStrLen); #print "Path is $pathStr\n"; $pathLen -= $pathStrLen; } while($pathLen); #print "done path!\n"; } sub processCookie { $cookieLen = &getNum($lenLen); do { $cookPart = &getNum($tagLen); $cookieLen -= $tagLen; # Strip any flag value $bool = (($cookPart & 0x80) == 0x80); $cookPart = $cookPart & 0x7f; if($cookPart == 0x10) { # Cookie name $nameLen = &getNum($lenLen); $cookieLen -= $lenLen; sysread(C, $cookName, $nameLen); #print "Cookie named $cookName\n"; $cookieLen -= $nameLen; } elsif($cookPart == 0x11) { # Cookie value $valLen = &getNum($lenLen); $cookieLen -= $lenLen; sysread(C, $cookValue, $valLen); #print "Cookie valued $cookValue\n"; $cookieLen -= $valLen; if($sa && $cookName eq "bbuserid") { $uid = $cookValue; } if($sa && $cookName eq "bbpassword") { $pass = $cookValue; } } elsif($cookPart == 0x12) { # Cookie expiry date $expireLen = &getNum($lenLen); $cookieLen -= $lenLen; $expireTime = &getNum($expireLen); $cookieLen -= $expireLen; #print "Cookie expires on ", scalar(localtime($expireTime)),"\n"; } elsif($cookPart == 0x13) { # Cookie last usage time $usedLen = &getNum($lenLen); $cookieLen -= $lenLen; $usedTime = &getNum($usedLen); $cookieLen -= $usedLen; #print "Cookie last used on ", scalar(localtime($usedTime)),"\n"; } elsif($cookPart == 0x19) { # HTTPS only } elsif($cookPart == 0x1B) { #print "Cookie will be sent to "; } elsif($cookPart == 0x1C) { # Delete protection } elsif($cookPart == 0x20) { # Not sent if /pathxyz, only if /path/xyz } elsif($cookPart == 0x22) { # set by login form } elsif($cookPart == 0x23) { # set by HTTP auth login } elsif($cookPart == 0x24) { # Third-party cookie } elsif($cookPart == 0x27) { # Mystery new Opera 9.50 flag } else { #print "Unexpected cookie type $cookPart\n"; exit; } } while($cookieLen); #print "done cookie!\n"; } __END__ Revision History 2010/09/25 -

is now

2008/12/01 - Removed fromearth.net reference from $cssURL 2008/09/23 - Added Viddler as a potential video source Pass forums referer to paintedover to get full-size image forumimages.somethingawful.com is now fi.somethingawful.com 2008/04/01 - Smileys are now stored along with the LP 2008/03/28 - Removed all code relating to archives.somethingawful.com Added blip.tv as an 'other' video site, removed stage6 2008/02/20 - Changed 'Date Collected' to 'Date Added' 2008/02/04 - Added code to make wget work more automatically on Mac OS X 2008/01/03 - Disable $downloadVideos by default 2007/12/31 - Strip trailing #..... stuff from images (eg #via=salr) 2007/12/23 - Line endings in output files will now be fixed to match the downloader's OS 2007/12/13 - Retry fetching any failed videos at the end of the script 2007/12/11 - Create stats.txt file with the total video size and any errors generated during the fetch 2007/12/05 - Automatically replace spaces and %xx characters in filenames with _s 2007/11/26 - Add ' | ' to video title in first column 2007/11/23 - Video links are renamed to the name of the video site for multiBackups 2007/11/20 - Support for multiple video mirrors on a single line Replaced downloadFiles and imagesOnly with downloadVideos 2007/11/17 - Selected smilies are replaced last 2007/11/14 - Added imagesOnly option 2007/11/12 - Added variable for extra parameters to wget, starting with setting the max download retries to 3 2007/11/10 - Various features copied over from fetch-thread: downloadFiles, keeping original image filenames, external smiley list, reporting fetch time 2007/11/08 - Last Post date is always '?' 2007/11/05 - Date Collected is auto-filled with today's date 2007/10/27 - Handle