From 08cee7066218cfa262616d91bf9e1f5297e19091 Mon Sep 17 00:00:00 2001 From: Cameron Ball Date: Sat, 21 Feb 2015 15:25:00 +0800 Subject: [PATCH] Better cache and upload stuff --- .gitignore | 3 +- Controllers/SimfileCacheController.php | 10 ++- Controllers/SimfileController.php | 94 +++++++++++++++++-------- Controllers/UserAuthController.php | 2 +- Services/CacheUpdater.php | 33 +++++++++ Services/ICacheUpdater.php | 11 +++ Services/IUserSession.php | 19 +++--- Services/Uploads/IUploadQueueManager.php | 9 +++ Services/Uploads/UploadQueueManager.php | 114 +++++++++++++++++++++++++++++++ Services/UserSession.php | 103 +++++++++++++++------------- config/DI.php | 4 ++ config/Routes.php | 11 ++- 12 files changed, 322 insertions(+), 91 deletions(-) create mode 100644 Services/CacheUpdater.php create mode 100644 Services/ICacheUpdater.php create mode 100644 Services/Uploads/IUploadQueueManager.php create mode 100644 Services/Uploads/UploadQueueManager.php diff --git a/.gitignore b/.gitignore index aa4599a..03a467d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ files/banners/* FacebookApp.php app.php SimfileCache -upload.html \ No newline at end of file +upload.html +Users \ No newline at end of file diff --git a/Controllers/SimfileCacheController.php b/Controllers/SimfileCacheController.php index 2b3f692..5281bf5 100644 --- a/Controllers/SimfileCacheController.php +++ b/Controllers/SimfileCacheController.php @@ -31,10 +31,10 @@ class SimfileCacheController implements IDivineController public function indexAction() { $get = $this->_request->get(); - if(!$get['token']) throw new Exception('Token missing'); + if(!$get['cronToken']) throw new Exception('Token missing'); //TODO: I should make $req->get('token') give the element and $req->get() return the array maybe? - if($get['token'] !== $this->_configManager->getDirective('cacheToken')) throw new Exception ('Invalid token'); + if($get['cronToken'] !== $this->_configManager->getDirective('cronToken')) throw new Exception ('Invalid token'); $all_files = scandir('../SimfileCache'); natsort($all_files); @@ -72,6 +72,12 @@ class SimfileCacheController implements IDivineController unlink('../SimfileCache/' . $filename); } } + + usort($completeArray['packs'], function($a, $b) + { + return strcmp($a['title'], $b['title']); + }); + file_put_contents('../SimfileCache/simfiles.json',json_encode($completeArray)); } else { foreach($simfiles as $simfile) diff --git a/Controllers/SimfileController.php b/Controllers/SimfileController.php index 3f32c09..5842bbf 100644 --- a/Controllers/SimfileController.php +++ b/Controllers/SimfileController.php @@ -6,11 +6,13 @@ use Controllers\IDivineController; use Services\Http\IHttpResponse; use Services\Http\IHttpRequest; use Services\Uploads\IUploadManager; +use Services\Uploads\IUploadQueueManager; use Services\IUserSession; use Services\IZipParser; use Services\ISMOMatcher; use Services\IStatusReporter; use Services\IConfigManager; +use Services\ICacheUpdater; use DataAccess\StepMania\ISimfileRepository; use DataAccess\StepMania\IPackRepository; use DataAccess\IFileRepository; @@ -26,6 +28,8 @@ class SimfileController implements IDivineController private $_request; private $_response; private $_uploadManager; + private $_uploadQueueManager; + private $_cacheUpdater; private $_zipParser; private $_smoMatcher; private $_downloadRepository; @@ -37,6 +41,8 @@ class SimfileController implements IDivineController IHttpResponse $response, IHttpRequest $request, IUploadManager $uploadManager, + IUploadQueueManager $uploadQueueManager, + ICacheUpdater $cacheUpdater, ISimfileRepository $simfileRepository, IPackRepository $packRepository, IFileRepository $fileRepository, @@ -50,6 +56,8 @@ class SimfileController implements IDivineController $this->_response = $response; $this->_request = $request; $this->_uploadManager = $uploadManager; + $this->_uploadQueueManager = $uploadQueueManager; + $this->_cacheUpdater = $cacheUpdater; $this->_simfileRepository = $simfileRepository; $this->_packRepository = $packRepository; $this->_fileRepository = $fileRepository; @@ -137,38 +145,70 @@ class SimfileController implements IDivineController //be deleted from the filesystem and database. foreach($files as $file) { - $zipParser = $this->_zipParser; - $zipParser->parse($file); - - if(!$zipParser->simfiles()) $this->_statusReporter->error('That zip doesn\'t seem to have any simfiles in it.'); - - //save the actual zip in the db - $this->findAndAddSmoMirror($file); - $this->_fileRepository->save($file); - - if($zipParser->isPack()) - { - //XXX: Tricky! pack() uses packbuilder and so returns a new pack each time. - //I tried to be clever and call pack() multiple times thinking I was getting the same - //object. Should I cache it in zipparser? - $pack = $zipParser->pack(); - $packBanner = $pack->getBanner() ? $this->_fileRepository->save($pack->getBanner()) : null; - $this->_packRepository->save($pack); - } - - foreach($zipParser->simfiles() as $simfile) - { - $banner = $simfile->getBanner() ? $this->_fileRepository->save($simfile->getBanner()) : null; - $simfileZip = $simfile->getSimfile() ? $this->_fileRepository->save($simfile->getSimfile()) : null; - - if(isset($pack)) $simfile->addToPack($pack); - $this->_simfileRepository->save($simfile); - } + $this->saveToDB($file); + } + + //Packs are inserted in saveToDB, only update here after they are all done. + $this->_cacheUpdater->update(); + $this->_statusReporter->success('Uploaded succesfully'); + } + + public function processUploadQueueAction() + { + $get = $this->_request->get(); + if(!$get['cronToken']) throw new Exception('Token missing'); + + //TODO: I should make $req->get('token') give the element and $req->get() return the array maybe? + if($get['cronToken'] !== $this->_configManager->getDirective('cronToken')) throw new Exception ('Invalid token'); + + $files = $this->_uploadQueueManager->setFilesDirectory($this->_configManager->getDirective('filesPath')) + ->setDestination('StepMania') + ->process(5); //TODO: Num to process should be config'd + + foreach($files as $uid => $file) + { + $this->_userSession->setCurrentUser($uid); + $this->saveToDB($file); } + //Packs are inserted in saveToDB, only update here after they are all done. + $this->_cacheUpdater->update(); $this->_statusReporter->success('Uploaded succesfully'); } + + private function saveToDB(IFile $file) + { + $zipParser = $this->_zipParser; + $zipParser->parse($file); + + if(!$zipParser->simfiles()) $this->_statusReporter->error('That zip doesn\'t seem to have any simfiles in it.'); + + //save the actual zip in the db + $this->findAndAddSmoMirror($file); + $this->_fileRepository->save($file); + + if($zipParser->isPack()) + { + //XXX: Tricky! pack() uses packbuilder and so returns a new pack each time. + //I tried to be clever and call pack() multiple times thinking I was getting the same + //object. Should I cache it in zipparser? + $pack = $zipParser->pack(); + $packBanner = $pack->getBanner() ? $this->_fileRepository->save($pack->getBanner()) : null; + $this->_packRepository->save($pack); + $this->_cacheUpdater->insert($pack); + } + + foreach($zipParser->simfiles() as $simfile) + { + $banner = $simfile->getBanner() ? $this->_fileRepository->save($simfile->getBanner()) : null; + $simfileZip = $simfile->getSimfile() ? $this->_fileRepository->save($simfile->getSimfile()) : null; + + if(isset($pack)) $simfile->addToPack($pack); + $this->_simfileRepository->save($simfile); + } + } + private function findAndAddSmoMirror(IFile $file) { $basename = pathinfo($file->getFilename(), PATHINFO_FILENAME); diff --git a/Controllers/UserAuthController.php b/Controllers/UserAuthController.php index af257b1..65d28b9 100644 --- a/Controllers/UserAuthController.php +++ b/Controllers/UserAuthController.php @@ -102,7 +102,7 @@ class UserAuthController implements IDivineController ->With_Name(new \Domain\VOs\Name($firstName, $lastName)) ->With_Tags(array()) ->With_FacebookId($facebookId) - ->With_Quota(100000000) //XXX: quota is in bytes + ->With_Quota(500000000) //XXX: quota is in bytes //XXX: Is this confusing? Maybe better to do a conditional and only call with_country when we have a country ->With_Country(isset($country) ? new \Domain\VOs\Country($country) : null) ->build(); diff --git a/Services/CacheUpdater.php b/Services/CacheUpdater.php new file mode 100644 index 0000000..a913df3 --- /dev/null +++ b/Services/CacheUpdater.php @@ -0,0 +1,33 @@ +_json = json_decode(file_get_contents('../SimfileCache/simfiles.json'), true); + } + + public function insert(IPack $pack) + { + $this->_json['packs'][] = Util::packToArray($pack); + return $this; + } + + public function update() + { + usort($this->_json['packs'], function($a, $b) + { + return strcmp($a['title'], $b['title']); + }); + + file_put_contents('../SimfileCache/simfiles.json',json_encode($this->_json)); + } +} + diff --git a/Services/ICacheUpdater.php b/Services/ICacheUpdater.php new file mode 100644 index 0000000..d7952f0 --- /dev/null +++ b/Services/ICacheUpdater.php @@ -0,0 +1,11 @@ +_fileFactory = $fileFactory; + $this->_fileBuilder = $builder; + + $files = new RecursiveIteratorIterator(new RecursiveDirectoryIterator('../Users', RecursiveIteratorIterator::SELF_FIRST)); + foreach($files as $file) + { + + //$parts = explode('/', $file->getPath()); + $dir = basename(dirname(($file->getPath()))); + if($file->getExtension() == 'zip' && $dir == 'Users') + { + $finfo = new finfo(FILEINFO_MIME); + $mimetype = $finfo->file($file->getPath() . '/' . $file->getFileName()); + //basename($file->getPath()) = the user's foldername, which is their ID + $this->_files[basename($file->getPath())] = $this->_fileFactory->createInstance( + $file->getFilename(), + $mimetype, + $file->getPath() . '/' . $file->getFilename(), + $file->getSize() + ); + } + } + } + + public function setFilesDirectory($path) { + if(!$this->destinationExists($path)) + { + throw new Exception('Invalid path. Path does not exist.'); + } + + $this->_basePath = $path; + + return $this; + } + + public function setDestination($path) { + if(!$this->destinationExists($this->_basePath . '/' . $path)) + { + throw new Exception('Invalid path. Path does not exist.'); + } + + $this->_destination = $path; + + return $this; + } + + private function destinationExists($path) { + return file_exists($path); + } + + private function saveFile(IFile $file) + { + if($this->_destination) + { + $randomName = md5_file($file->getTempName()); + //$result = move_uploaded_file($file->getTempName(), $this->_basePath . '/' . $this->_destination . '/' . $randomName . '.' . $file->getExtension()); + $result = rename($file->getTempName(), $this->_basePath . '/' . $this->_destination . '/' . $randomName . '.' . $file->getExtension()); + } + + if(!$result) + { + throw new Exception("Could not save file."); + } + + return $randomName; + } + + public function process($num) + { + $results = array(); + $i = 0; + foreach($this->_files as $uid => $file) + { + $hash = $this->saveFile($file); + + /* @var $file \Services\Uploads\IFile */ + $results[$uid] = $this->_fileBuilder->With_Hash($hash) + ->With_Path(rtrim($this->_destination, '/')) + ->With_Filename($file->getName()) + ->With_Mimetype($file->getType()) + ->With_Size($file->getSize()) + ->With_UploadDate(time()) + ->build(); + $i++; + if($i == $num) break; + } + + return $results; + } +} diff --git a/Services/UserSession.php b/Services/UserSession.php index ead323f..040333d 100644 --- a/Services/UserSession.php +++ b/Services/UserSession.php @@ -1,50 +1,55 @@ -_request = $request; - $this->_userRepository = $repository; - - $token = $this->findToken(); - $this->_currentUser = $token ? $this->_userRepository->findByAuthToken($token) : null; - } - - public function getCurrentUser() - { - return $this->_currentUser; - } - - private function findToken() - { - //if($this->_request->isPost()) - //{ - //Try post - $request = $this->_request->post(); - if(!empty($request['token'])) return $request['token']; - //} - - //if($this->_request->isGet()) - //{ - //Try get - $request = $this->_request->get(); - if(!empty($request['token'])) return $request['token']; - //} - - //no good, try the body - $body = json_decode($this->_request->getBody(), true); - if(!empty($body['token'])) return $body['token']; - - return null; - } +_request = $request; + $this->_userRepository = $repository; + + $token = $this->findToken(); + $this->_currentUser = $token ? $this->_userRepository->findByAuthToken($token) : null; + } + + public function setCurrentUser($uid) + { + $this->_currentUser = $this->_userRepository->findById($uid); + } + + public function getCurrentUser() + { + return $this->_currentUser; + } + + private function findToken() + { + //if($this->_request->isPost()) + //{ + //Try post + $request = $this->_request->post(); + if(!empty($request['token'])) return $request['token']; + //} + + //if($this->_request->isGet()) + //{ + //Try get + $request = $this->_request->get(); + if(!empty($request['token'])) return $request['token']; + //} + + //no good, try the body + $body = json_decode($this->_request->getBody(), true); + if(!empty($body['token'])) return $body['token']; + + return null; + } } \ No newline at end of file diff --git a/config/DI.php b/config/DI.php index 76398f5..f09a537 100644 --- a/config/DI.php +++ b/config/DI.php @@ -33,6 +33,10 @@ return [ 'Services\Routing\IRouter' => DI\object('Services\Routing\Router') ->constructor(DI\link('router.maps')), 'Services\Uploads\IUploadManager' => DI\object('Services\Uploads\UploadManager'), + 'Services\Uploads\IUploadQueueManager' => DI\object('Services\Uploads\UploadQueueManager') + ->lazy(), + 'Services\ICacheUpdater' => DI\object('Services\CacheUpdater') + ->lazy(), 'Services\IUserSession' => DI\object('Services\UserSession'), 'Services\IUserQuota' => DI\object('Services\UserQuota'), 'Services\Uploads\IFileFactory' => DI\object('Services\Uploads\FileFactory'), diff --git a/config/Routes.php b/config/Routes.php index 7e3afb7..39fa2ff 100644 --- a/config/Routes.php +++ b/config/Routes.php @@ -29,10 +29,17 @@ return [ ] ], - '/simfiles/upload' => [ + //'/simfiles/upload' => [ + // 'controller' => 'Simfile', + // 'actions' => [ + // 'POST' => 'upload' + // ] + //], + + '/simfiles/upload/process' => [ 'controller' => 'Simfile', 'actions' => [ - 'POST' => 'upload' + 'GET' => 'processUploadQueue' ] ], -- 2.11.0