From 161aab9f4e71fe0c4810a60acf867c0b82fc4cdf Mon Sep 17 00:00:00 2001 From: Cameron Ball Date: Thu, 27 Nov 2014 15:51:19 +0800 Subject: [PATCH] Files can now have mirrors. Implemented SMOMatcher service to attempt to automatically match packs against SMO. --- Controllers/SimfileController.php | 36 +++++++++++++-- Domain/Entities/File.php | 26 ++++++++++- Domain/Entities/FileBuilder.php | 9 +++- Domain/Entities/FileFactory.php | 19 ++++---- Domain/Entities/FileStepByStepBuilder.php | 6 +++ Domain/Entities/IFile.php | 4 ++ Domain/Entities/IFileBuilder.php | 1 + Domain/VOs/FileMirror.php | 21 +++++++++ Domain/VOs/IFileMirror.php | 8 ++++ Services/ISMOMatcher.php | 8 ++++ Services/SMOMatcher.php | 74 +++++++++++++++++++++++++++++++ config/DI.php | 1 + config/DataMaps.php | 11 ++++- 13 files changed, 209 insertions(+), 15 deletions(-) create mode 100644 Domain/VOs/FileMirror.php create mode 100644 Domain/VOs/IFileMirror.php create mode 100644 Services/ISMOMatcher.php create mode 100644 Services/SMOMatcher.php diff --git a/Controllers/SimfileController.php b/Controllers/SimfileController.php index 28a3061..d73527d 100644 --- a/Controllers/SimfileController.php +++ b/Controllers/SimfileController.php @@ -7,11 +7,13 @@ use Services\Http\IHttpResponse; use Services\Uploads\IUploadManager; use Services\IUserSession; use Services\IZipParser; +use Services\ISMOMatcher; use DataAccess\StepMania\ISimfileRepository; use DataAccess\StepMania\IPackRepository; use DataAccess\IFileRepository; use Domain\Entities\StepMania\ISimfile; -use Domain\Entities\StepMania\IPack; +use Domain\Entities\IFile; +use Domain\VOs\IFileMirror; class SimfileController implements IDivineController { @@ -22,6 +24,7 @@ class SimfileController implements IDivineController private $_uploadManager; private $_userSession; private $_zipParser; + private $_smoMatcher; public function __construct( IHttpResponse $response, @@ -30,7 +33,8 @@ class SimfileController implements IDivineController IPackRepository $packRepository, IFileRepository $fileRepository, IUserSession $userSession, - IZipParser $zipParser + IZipParser $zipParser, + ISMOMatcher $smoMatcher ) { $this->_response = $response; $this->_uploadManager = $uploadManager; @@ -39,6 +43,7 @@ class SimfileController implements IDivineController $this->_fileRepository = $fileRepository; $this->_userSession = $userSession; $this->_zipParser = $zipParser; + $this->_smoMatcher = $smoMatcher; } public function indexAction() { @@ -67,11 +72,21 @@ class SimfileController implements IDivineController $packSimfiles[] = $this->simfileToArray($simfile); } + $packMirrors = array(); + if($pack->getFile()->getMirrors()) + { + foreach($pack->getFile()->getMirrors() as $mirror) + { + $packMirrors = $mirror->getUri(); + } + } + $packArray[] = array( 'title'=> $pack->getTitle(), 'contributors' => $pack->getContributors(), 'simfiles' => $packSimfiles, - 'banner' => $pack->getBanner() ? 'files/banner/' . $pack->getBanner()->getHash() : 'files/banner/default' + 'banner' => $pack->getBanner() ? 'files/banner/' . $pack->getBanner()->getHash() : 'files/banner/default', + 'mirrors' => $packMirrors ); } @@ -93,8 +108,9 @@ class SimfileController implements IDivineController { $zipParser = $this->_zipParser; $zipParser->parse($file); - + //save the actual zip in the db + $this->findAndAddSmoMirror($file); $this->_fileRepository->save($file); if($zipParser->isPack()) @@ -118,6 +134,18 @@ class SimfileController implements IDivineController } } + private function findAndAddSmoMirror(IFile $file) + { + $basename = pathinfo($file->getFilename(), PATHINFO_FILENAME); + $match = $this->_smoMatcher->match($basename, $file->getSize()); + + //XXX: Direct instantiation of FileMirror bad? + if($match && $match['confidence'] > 90) + { + $file->addMirror(new \Domain\VOs\FileMirror($match['href'])); + } + } + private function simfileToArray(ISimfile $simfile) { $singleSteps = array(); diff --git a/Domain/Entities/File.php b/Domain/Entities/File.php index 3b70b35..8f6e1a3 100644 --- a/Domain/Entities/File.php +++ b/Domain/Entities/File.php @@ -3,6 +3,7 @@ namespace Domain\Entities; use Domain\Entities\AbstractEntity; +use Domain\VOs\IFileMirror; class File extends AbstractEntity implements IFile { @@ -12,6 +13,7 @@ class File extends AbstractEntity implements IFile private $_mimetype; private $_size; private $_uploadDate; + private $_mirrors; public function __construct( $hash, @@ -19,7 +21,8 @@ class File extends AbstractEntity implements IFile $filename, $mimetype, $size, - $uploadDate + $uploadDate, + array $mirrors = null ) { $this->_hash = $hash; $this->_path = $path; @@ -27,6 +30,17 @@ class File extends AbstractEntity implements IFile $this->_mimetype = $mimetype; $this->_size = $size; $this->_uploadDate = $uploadDate; + + if($mirrors) + { + foreach($mirrors as $mirror) { + if(!$mirror instanceof IFileMirror) { + throw new InvalidStepChartException(sprintf('Invalid FileMirror array. All array elements must be an instance of IFileMirror.')); + } + } + + $this->_mirrors = $mirrors; + } } public function getFilename() @@ -58,4 +72,14 @@ class File extends AbstractEntity implements IFile { return $this->_uploadDate; } + + public function getMirrors() + { + return $this->_mirrors; + } + + public function addMirror(IFileMirror $mirror) + { + $this->_mirrors[] = $mirror; + } } \ No newline at end of file diff --git a/Domain/Entities/FileBuilder.php b/Domain/Entities/FileBuilder.php index d67447b..407e14e 100644 --- a/Domain/Entities/FileBuilder.php +++ b/Domain/Entities/FileBuilder.php @@ -13,6 +13,7 @@ class FileBuilder implements IFileBuilder private $_mimetype; private $_size; private $_date; + private $_mirrors; public function __construct(IFileFactory $fileFactory) { @@ -49,6 +50,11 @@ class FileBuilder implements IFileBuilder $this->_date = $date; } + public function With_Mirrors(array $mirrors = null) + { + $this->_mirrors = $mirrors; + } + public function build() { return $this->_fileFactory @@ -58,6 +64,7 @@ class FileBuilder implements IFileBuilder $this->_filename, $this->_mimetype, $this->_size, - $this->_date); + $this->_date, + $this->_mirrors); } } \ No newline at end of file diff --git a/Domain/Entities/FileFactory.php b/Domain/Entities/FileFactory.php index dd66810..ce9cb22 100644 --- a/Domain/Entities/FileFactory.php +++ b/Domain/Entities/FileFactory.php @@ -12,7 +12,8 @@ interface IFileFactory $filename, $mimetype, $size, - $uploadDate + $uploadDate, + array $mirrors = null ); } @@ -24,15 +25,17 @@ class FileFactory implements IFileFactory $filename, $mimetype, $size, - $uploadDate + $uploadDate, + array $mirrors = null ) { return new File( - $hash, - $path, - $filename, - $mimetype, - $size, - $uploadDate + $hash, + $path, + $filename, + $mimetype, + $size, + $uploadDate, + $mirrors ); } } diff --git a/Domain/Entities/FileStepByStepBuilder.php b/Domain/Entities/FileStepByStepBuilder.php index 1f22a6b..a76dfd6 100644 --- a/Domain/Entities/FileStepByStepBuilder.php +++ b/Domain/Entities/FileStepByStepBuilder.php @@ -36,6 +36,7 @@ interface IFileStepByStepBuilder_With_Size interface IFileStepByStepBuilder_With_UploadDate { + public function With_Mirrors(array $mirrors = null); public function build(); } @@ -100,6 +101,11 @@ class FileStepByStepBuilder_With_Size extends AbstractFileStepByStepBuilder impl class FileStepByStepBuilder_With_UploadDate extends AbstractFileStepByStepBuilder implements IFileStepByStepBuilder_With_UploadDate { + public function With_Mirrors(array $mirrors = null) { + $this->_fileBuilder->With_Mirrors($mirrors); + return new FileStepByStepBuilder_With_UploadDate($this->_fileBuilder); + } + public function build() { return $this->_fileBuilder ->build(); diff --git a/Domain/Entities/IFile.php b/Domain/Entities/IFile.php index 4ce49c3..458b359 100644 --- a/Domain/Entities/IFile.php +++ b/Domain/Entities/IFile.php @@ -2,6 +2,8 @@ namespace Domain\Entities; +use Domain\VOs\IFileMirror; + interface IFile extends IDivineEntity { public function getPath(); @@ -10,4 +12,6 @@ interface IFile extends IDivineEntity public function getMimetype(); public function getSize(); public function getUploadDate(); + public function getMirrors(); + public function addMirror(IFileMirror $mirror); } \ No newline at end of file diff --git a/Domain/Entities/IFileBuilder.php b/Domain/Entities/IFileBuilder.php index 95b7ab4..b014b5f 100644 --- a/Domain/Entities/IFileBuilder.php +++ b/Domain/Entities/IFileBuilder.php @@ -10,5 +10,6 @@ interface IFileBuilder public function With_Mimetype($mimetype); public function With_Size($size); public function With_UploadDate($date); + public function With_Mirrors(array $mirrors = null); public function build(); } \ No newline at end of file diff --git a/Domain/VOs/FileMirror.php b/Domain/VOs/FileMirror.php new file mode 100644 index 0000000..6e991f0 --- /dev/null +++ b/Domain/VOs/FileMirror.php @@ -0,0 +1,21 @@ +_uri = $uri; + } + + public function getUri() + { + return $this->_uri; + } +} + diff --git a/Domain/VOs/IFileMirror.php b/Domain/VOs/IFileMirror.php new file mode 100644 index 0000000..3892be4 --- /dev/null +++ b/Domain/VOs/IFileMirror.php @@ -0,0 +1,8 @@ +scrapeSmo()) return null; + + $most_likely = array('confidence' => 0); + foreach($this->_records as $rowNum => $row) { + $cells = $row->getElementsByTagName('td'); + + if($rowNum > 0) { + $candidate = $this->cellToCandidateArray($cells); + $this->setCandidateSimilarty($title, $filesize, $candidate); + $most_likely = $candidate['confidence'] > $most_likely['confidence'] ? $candidate : $most_likely; + } + } + + return $most_likely; + } + + private function scrapeSmo() + { + if($this->_records) return true; + + $c = curl_init('http://stepmaniaonline.net/index.php?page=downloads'); + curl_setopt($c, CURLOPT_RETURNTRANSFER, true); + //curl_setopt(... other options you want...) + + $html = curl_exec($c); + + if (curl_error($c)) return false; + + // Get the status code + $status = curl_getinfo($c, CURLINFO_HTTP_CODE); + + curl_close($c); + //$html = file_get_contents('smo.html'); + $dom = new DOMDocument(); + @$dom->loadHTML($html); + $xpath = new DOMXPath($dom); + + $table = $xpath->query("/html/body/div[@class='container']/div[@class='mid']/div[@class='content']/div[3]/div[@class='blockcontent']/table")->item(0); + $this->_records = $table->getElementsByTagName("tr"); + + return true; + } + + private function cellToCandidateArray(DOMNodeList $cells) + { + return array( + 'href' => $cells->item(0)->getElementsByTagName('a')->item(0)->getAttribute('href'), + 'title' => $cells->item(0)->nodeValue, + 'filesize' => strpos($cells->item(1)->nodeValue, 'Gb') === true ? ($cells->item(1)->nodeValue)*1024*1024*1024 : ($cells->item(1)->nodeValue)*1024*1024 + ); + } + + private function setCandidateSimilarty($title, $filesize, array &$candidate) + { + similar_text($title, $candidate['title'], $percent); + $r = $candidate['filesize'] > $filesize ? $filesize/$candidate['filesize'] : $candidate['filesize']/$filesize; + $candidate['confidence'] = $percent*$r; + } +} \ No newline at end of file diff --git a/config/DI.php b/config/DI.php index 574d4c1..3d117d0 100644 --- a/config/DI.php +++ b/config/DI.php @@ -37,6 +37,7 @@ return [ 'Services\ISimfileParser' => DI\object('Services\SimfileParser'), 'Services\IZipParser' => DI\object('Services\ZipParser'), 'Services\IBannerExtracter' => DI\object('Services\BannerExtracter'), + 'Services\ISMOMatcher' => DI\object('Services\SMOMatcher'), //DA 'DataAccess\StepMania\ISimfileRepository' => DI\object('DataAccess\StepMania\SimfileRepository'), diff --git a/config/DataMaps.php b/config/DataMaps.php index a647632..358ccc6 100644 --- a/config/DataMaps.php +++ b/config/DataMaps.php @@ -137,7 +137,16 @@ return [ 'filename' => DataAccess\Varchar('filename'), 'mimetype' => DataAccess\Varchar('mimetype'), 'size' => DataAccess\Int('size'), - 'uploadDate' => DataAccess\Int('uploaded', 'getUploadDate') + 'uploadDate' => DataAccess\Int('uploaded', 'getUploadDate'), + 'mirrors' => DataAccess\VOArray('FileMirror', 'getMirrors') + ] + ], + + 'FileMirror' => [ + 'class' => 'Domain\VOs\FileMirror', + 'table' => 'mirrors', + 'maps' => [ + 'uri' => DataAccess\Varchar('uri') ] ] ]; -- 2.11.0