Better cache and upload stuff
authorCameron Ball <c.ball1729@gmail.com>
Sat, 21 Feb 2015 07:25:00 +0000 (15:25 +0800)
committerCameron Ball <c.ball1729@gmail.com>
Sat, 21 Feb 2015 07:25:00 +0000 (15:25 +0800)
12 files changed:
.gitignore
Controllers/SimfileCacheController.php
Controllers/SimfileController.php
Controllers/UserAuthController.php
Services/CacheUpdater.php [new file with mode: 0644]
Services/ICacheUpdater.php [new file with mode: 0644]
Services/IUserSession.php
Services/Uploads/IUploadQueueManager.php [new file with mode: 0644]
Services/Uploads/UploadQueueManager.php [new file with mode: 0644]
Services/UserSession.php
config/DI.php
config/Routes.php

index aa4599a..03a467d 100644 (file)
@@ -10,4 +10,5 @@ files/banners/*
 FacebookApp.php
 app.php
 SimfileCache
-upload.html
\ No newline at end of file
+upload.html
+Users
\ No newline at end of file
index 2b3f692..5281bf5 100644 (file)
@@ -31,10 +31,10 @@ class SimfileCacheController implements IDivineController
     
     public function indexAction() {
         $get = $this->_request->get();
-        if(!$get['token']) throw new Exception('Token missing');
+        if(!$get['cronToken']) throw new Exception('Token missing');
         
         //TODO: I should make $req->get('token') give the element and $req->get() return the array maybe?
-        if($get['token'] !== $this->_configManager->getDirective('cacheToken')) throw new Exception ('Invalid token');
+        if($get['cronToken'] !== $this->_configManager->getDirective('cronToken')) throw new Exception ('Invalid token');
         
         $all_files = scandir('../SimfileCache');
         natsort($all_files);
@@ -72,6 +72,12 @@ class SimfileCacheController implements IDivineController
                 unlink('../SimfileCache/' . $filename);
                 }
             }
+            
+            usort($completeArray['packs'], function($a, $b)
+            {
+                return strcmp($a['title'], $b['title']);
+            });
+            
             file_put_contents('../SimfileCache/simfiles.json',json_encode($completeArray));
         } else {
             foreach($simfiles as $simfile)
index 3f32c09..5842bbf 100644 (file)
@@ -6,11 +6,13 @@ use Controllers\IDivineController;
 use Services\Http\IHttpResponse;\r
 use Services\Http\IHttpRequest;\r
 use Services\Uploads\IUploadManager;\r
+use Services\Uploads\IUploadQueueManager;\r
 use Services\IUserSession;\r
 use Services\IZipParser;\r
 use Services\ISMOMatcher;\r
 use Services\IStatusReporter;\r
 use Services\IConfigManager;\r
+use Services\ICacheUpdater;\r
 use DataAccess\StepMania\ISimfileRepository;\r
 use DataAccess\StepMania\IPackRepository;\r
 use DataAccess\IFileRepository;\r
@@ -26,6 +28,8 @@ class SimfileController implements IDivineController
     private $_request;\r
     private $_response;\r
     private $_uploadManager;\r
+    private $_uploadQueueManager;\r
+    private $_cacheUpdater;\r
     private $_zipParser;\r
     private $_smoMatcher;\r
     private $_downloadRepository;\r
@@ -37,6 +41,8 @@ class SimfileController implements IDivineController
         IHttpResponse $response,\r
         IHttpRequest $request,\r
         IUploadManager $uploadManager,\r
+        IUploadQueueManager $uploadQueueManager,\r
+        ICacheUpdater $cacheUpdater,\r
         ISimfileRepository $simfileRepository,\r
         IPackRepository $packRepository,\r
         IFileRepository $fileRepository,\r
@@ -50,6 +56,8 @@ class SimfileController implements IDivineController
         $this->_response = $response;\r
         $this->_request = $request;\r
         $this->_uploadManager = $uploadManager;\r
+        $this->_uploadQueueManager = $uploadQueueManager;\r
+        $this->_cacheUpdater = $cacheUpdater;\r
         $this->_simfileRepository = $simfileRepository;\r
         $this->_packRepository = $packRepository;\r
         $this->_fileRepository = $fileRepository;\r
@@ -137,38 +145,70 @@ class SimfileController implements IDivineController
         //be deleted from the filesystem and database.\r
         foreach($files as $file)\r
         {\r
-            $zipParser = $this->_zipParser;\r
-            $zipParser->parse($file);\r
-\r
-            if(!$zipParser->simfiles()) $this->_statusReporter->error('That zip doesn\'t seem to have any simfiles in it.');\r
-\r
-            //save the actual zip in the db\r
-            $this->findAndAddSmoMirror($file);\r
-            $this->_fileRepository->save($file);  \r
-            \r
-            if($zipParser->isPack())\r
-            {\r
-                //XXX: Tricky! pack() uses packbuilder and so returns a new pack each time.\r
-                //I tried to be clever and call pack() multiple times thinking I was getting the same\r
-                //object. Should I cache it in zipparser?\r
-                $pack = $zipParser->pack();\r
-                $packBanner = $pack->getBanner() ? $this->_fileRepository->save($pack->getBanner()) : null;\r
-                $this->_packRepository->save($pack);\r
-            }\r
-            \r
-            foreach($zipParser->simfiles() as $simfile)\r
-            {   \r
-                $banner = $simfile->getBanner() ? $this->_fileRepository->save($simfile->getBanner()) : null;\r
-                $simfileZip = $simfile->getSimfile() ? $this->_fileRepository->save($simfile->getSimfile()) : null;\r
-\r
-                if(isset($pack)) $simfile->addToPack($pack);\r
-                $this->_simfileRepository->save($simfile);\r
-            }\r
+            $this->saveToDB($file);\r
+        }\r
+        \r
+        //Packs are inserted in saveToDB, only update here after they are all done.\r
+        $this->_cacheUpdater->update();\r
+        $this->_statusReporter->success('Uploaded succesfully');\r
+    }\r
+    \r
+    public function processUploadQueueAction()\r
+    {\r
+        $get = $this->_request->get();\r
+        if(!$get['cronToken']) throw new Exception('Token missing');\r
+        \r
+        //TODO: I should make $req->get('token') give the element and $req->get() return the array maybe?\r
+        if($get['cronToken'] !== $this->_configManager->getDirective('cronToken')) throw new Exception ('Invalid token');\r
+        \r
+        $files = $this->_uploadQueueManager->setFilesDirectory($this->_configManager->getDirective('filesPath'))\r
+                                           ->setDestination('StepMania')\r
+                                           ->process(5); //TODO: Num to process should be config'd\r
+        \r
+        foreach($files as $uid => $file)\r
+        {\r
+            $this->_userSession->setCurrentUser($uid);\r
+            $this->saveToDB($file);\r
         }\r
         \r
+        //Packs are inserted in saveToDB, only update here after they are all done.\r
+        $this->_cacheUpdater->update();\r
         $this->_statusReporter->success('Uploaded succesfully');\r
     }\r
             \r
+       \r
+    private function saveToDB(IFile $file)\r
+    {\r
+        $zipParser = $this->_zipParser;\r
+        $zipParser->parse($file);\r
+\r
+        if(!$zipParser->simfiles()) $this->_statusReporter->error('That zip doesn\'t seem to have any simfiles in it.');\r
+\r
+        //save the actual zip in the db\r
+        $this->findAndAddSmoMirror($file);\r
+        $this->_fileRepository->save($file);  \r
+\r
+        if($zipParser->isPack())\r
+        {\r
+            //XXX: Tricky! pack() uses packbuilder and so returns a new pack each time.\r
+            //I tried to be clever and call pack() multiple times thinking I was getting the same\r
+            //object. Should I cache it in zipparser?\r
+            $pack = $zipParser->pack();\r
+            $packBanner = $pack->getBanner() ? $this->_fileRepository->save($pack->getBanner()) : null;\r
+            $this->_packRepository->save($pack);\r
+            $this->_cacheUpdater->insert($pack);\r
+        }\r
+\r
+        foreach($zipParser->simfiles() as $simfile)\r
+        {   \r
+            $banner = $simfile->getBanner() ? $this->_fileRepository->save($simfile->getBanner()) : null;\r
+            $simfileZip = $simfile->getSimfile() ? $this->_fileRepository->save($simfile->getSimfile()) : null;\r
+\r
+            if(isset($pack)) $simfile->addToPack($pack);\r
+            $this->_simfileRepository->save($simfile);\r
+        }\r
+    }\r
+    \r
     private function findAndAddSmoMirror(IFile $file)\r
     {\r
         $basename = pathinfo($file->getFilename(), PATHINFO_FILENAME);\r
index af257b1..65d28b9 100644 (file)
@@ -102,7 +102,7 @@ class UserAuthController implements IDivineController
                                                 ->With_Name(new \Domain\VOs\Name($firstName, $lastName))
                                                 ->With_Tags(array())
                                                 ->With_FacebookId($facebookId)
-                                                ->With_Quota(100000000) //XXX: quota is in bytes
+                                                ->With_Quota(500000000) //XXX: quota is in bytes
                                                 //XXX: Is this confusing? Maybe better to do a conditional and only call with_country when we have a country
                                                 ->With_Country(isset($country) ? new \Domain\VOs\Country($country) : null)
                                                 ->build();
diff --git a/Services/CacheUpdater.php b/Services/CacheUpdater.php
new file mode 100644 (file)
index 0000000..a913df3
--- /dev/null
@@ -0,0 +1,33 @@
+<?php\r
+\r
+namespace Services;\r
+\r
+use Services\ICacheUpdater;\r
+use Domain\Entities\StepMania\IPack;\r
+use Domain\Util;\r
+\r
+class CacheUpdater implements ICacheUpdater\r
+{\r
+    private $_json;\r
+    \r
+    public function __construct() {\r
+        $this->_json = json_decode(file_get_contents('../SimfileCache/simfiles.json'), true);\r
+    }\r
+    \r
+    public function insert(IPack $pack)\r
+    {\r
+        $this->_json['packs'][] = Util::packToArray($pack);\r
+        return $this;\r
+    }\r
+    \r
+    public function update()\r
+    {\r
+        usort($this->_json['packs'], function($a, $b)\r
+        {\r
+            return strcmp($a['title'], $b['title']);\r
+        });\r
+            \r
+        file_put_contents('../SimfileCache/simfiles.json',json_encode($this->_json));\r
+    }\r
+}\r
+\r
diff --git a/Services/ICacheUpdater.php b/Services/ICacheUpdater.php
new file mode 100644 (file)
index 0000000..d7952f0
--- /dev/null
@@ -0,0 +1,11 @@
+<?php\r
+\r
+namespace Services;\r
+\r
+use Domain\Entities\StepMania\IPack;\r
+\r
+interface ICacheUpdater\r
+{\r
+    public function insert(IPack $pack);\r
+    public function update();\r
+}\r
index 96b13dd..bec5b6c 100644 (file)
@@ -1,9 +1,10 @@
-<?php
-
-namespace Services;
-
-interface IUserSession
-{
-    public function getCurrentUser();
-}
-
+<?php\r
+\r
+namespace Services;\r
+\r
+interface IUserSession\r
+{\r
+    public function getCurrentUser();\r
+    public function setCurrentUser($uid);\r
+}\r
+\r
diff --git a/Services/Uploads/IUploadQueueManager.php b/Services/Uploads/IUploadQueueManager.php
new file mode 100644 (file)
index 0000000..214d8d2
--- /dev/null
@@ -0,0 +1,9 @@
+<?php\r
+\r
+namespace Services\Uploads;\r
+\r
+interface IUploadQueueManager {\r
+    public function setFilesDirectory($path);\r
+    public function setDestination($path);\r
+    public function process($num);\r
+}\r
diff --git a/Services/Uploads/UploadQueueManager.php b/Services/Uploads/UploadQueueManager.php
new file mode 100644 (file)
index 0000000..4b2f138
--- /dev/null
@@ -0,0 +1,114 @@
+<?php\r
+\r
+namespace Services\Uploads;\r
+\r
+use Services\Uploads\IUploadQueueManager;\r
+use Services\Uploads\IFileFactory;\r
+use Services\Uploads\IFile;\r
+//This was a bit silly, I have a File object to use with the service but there is also a File entity.\r
+//It's confusing but if you pay attention it should be OK.\r
+use Domain\Entities\IFileStepByStepBuilder;\r
+use Exception;\r
+use RecursiveIteratorIterator;\r
+use RecursiveDirectoryIterator;\r
+use finfo;\r
+\r
+class UploadQueueManager implements IUploadQueueManager{        \r
+    \r
+    private $_files= array();\r
+    private $_fileFactory;\r
+    private $_basePath;\r
+    private $_destination;\r
+    private $_fileBuilder;\r
+        \r
+    public function __construct(IFileFactory $fileFactory, IFileStepByStepBuilder $builder) {\r
+        $this->_fileFactory = $fileFactory;\r
+        $this->_fileBuilder = $builder;\r
+        \r
+        $files = new RecursiveIteratorIterator(new RecursiveDirectoryIterator('../Users', RecursiveIteratorIterator::SELF_FIRST));\r
+        foreach($files as $file)\r
+        {\r
+            \r
+            //$parts = explode('/', $file->getPath());\r
+            $dir = basename(dirname(($file->getPath())));\r
+            if($file->getExtension() == 'zip' && $dir == 'Users')\r
+            {   \r
+                $finfo = new finfo(FILEINFO_MIME);\r
+                $mimetype = $finfo->file($file->getPath() . '/' . $file->getFileName());\r
+                //basename($file->getPath()) = the user's foldername, which is their ID\r
+                $this->_files[basename($file->getPath())] = $this->_fileFactory->createInstance(\r
+                    $file->getFilename(),\r
+                    $mimetype,\r
+                    $file->getPath() . '/' . $file->getFilename(),\r
+                    $file->getSize()\r
+                );\r
+            }\r
+        }\r
+    }\r
+    \r
+    public function setFilesDirectory($path) {\r
+        if(!$this->destinationExists($path))\r
+        {\r
+            throw new Exception('Invalid path. Path does not exist.');\r
+        }\r
+        \r
+        $this->_basePath = $path;\r
+        \r
+        return $this;\r
+    }\r
+    \r
+    public function setDestination($path) {\r
+        if(!$this->destinationExists($this->_basePath . '/' . $path))\r
+        {\r
+            throw new Exception('Invalid path. Path does not exist.');\r
+        }\r
+        \r
+        $this->_destination = $path;\r
+        \r
+        return $this;\r
+    }\r
+    \r
+    private function destinationExists($path) {\r
+        return file_exists($path);\r
+    }\r
+    \r
+    private function saveFile(IFile $file)\r
+    {\r
+        if($this->_destination)\r
+        {\r
+            $randomName = md5_file($file->getTempName());\r
+            //$result = move_uploaded_file($file->getTempName(), $this->_basePath . '/' . $this->_destination . '/' . $randomName . '.' . $file->getExtension());\r
+            $result = rename($file->getTempName(), $this->_basePath . '/' . $this->_destination . '/' . $randomName . '.' . $file->getExtension());\r
+        }\r
+        \r
+        if(!$result)\r
+        {\r
+            throw new Exception("Could not save file.");\r
+        }\r
+        \r
+        return $randomName;\r
+    }\r
+    \r
+    public function process($num)\r
+    {\r
+        $results = array();\r
+        $i = 0;\r
+        foreach($this->_files as $uid => $file)\r
+        {\r
+            $hash = $this->saveFile($file);\r
+            \r
+            /* @var $file \Services\Uploads\IFile */\r
+            $results[$uid] = $this->_fileBuilder->With_Hash($hash)\r
+                                            ->With_Path(rtrim($this->_destination, '/'))\r
+                                            ->With_Filename($file->getName())\r
+                                            ->With_Mimetype($file->getType())\r
+                                            ->With_Size($file->getSize())\r
+                                            ->With_UploadDate(time())\r
+                                            ->build();\r
+            $i++;\r
+            if($i == $num) break;\r
+        }\r
+        \r
+        return $results;\r
+    }\r
+}\r
index ead323f..040333d 100644 (file)
@@ -1,50 +1,55 @@
-<?php
-
-namespace Services;
-
-use Services\Http\IHttpRequest;
-use DataAccess\IUserRepository;
-
-class UserSession implements IUserSession
-{
-    private $_userRepository;
-    private $_request;
-    private $_currentUser;
-    
-    public function __construct(IHttpRequest $request, IUserRepository $repository)
-    {
-        $this->_request = $request;
-        $this->_userRepository = $repository;
-
-        $token = $this->findToken();
-        $this->_currentUser = $token ? $this->_userRepository->findByAuthToken($token) : null;
-    }
-    
-    public function getCurrentUser()
-    {
-        return $this->_currentUser;
-    }
-
-    private function findToken()
-    {        
-        //if($this->_request->isPost())
-        //{
-        //Try post
-            $request = $this->_request->post();
-            if(!empty($request['token'])) return $request['token'];
-        //}
-        
-        //if($this->_request->isGet())
-        //{
-        //Try get
-            $request = $this->_request->get();
-            if(!empty($request['token'])) return $request['token'];
-        //}
-        
-        //no good, try the body
-        $body = json_decode($this->_request->getBody(), true);
-        if(!empty($body['token'])) return $body['token'];
-        
-        return null;
-    }
+<?php\r
+\r
+namespace Services;\r
+\r
+use Services\Http\IHttpRequest;\r
+use DataAccess\IUserRepository;\r
+\r
+class UserSession implements IUserSession\r
+{\r
+    private $_userRepository;\r
+    private $_request;\r
+    private $_currentUser;\r
+    \r
+    public function __construct(IHttpRequest $request, IUserRepository $repository)\r
+    {\r
+        $this->_request = $request;\r
+        $this->_userRepository = $repository;\r
+\r
+        $token = $this->findToken();\r
+        $this->_currentUser = $token ? $this->_userRepository->findByAuthToken($token) : null;\r
+    }\r
+    \r
+    public function setCurrentUser($uid)\r
+    {\r
+        $this->_currentUser = $this->_userRepository->findById($uid);\r
+    }\r
+    \r
+    public function getCurrentUser()\r
+    {\r
+        return $this->_currentUser;\r
+    }\r
+\r
+    private function findToken()\r
+    {        \r
+        //if($this->_request->isPost())\r
+        //{\r
+        //Try post\r
+            $request = $this->_request->post();\r
+            if(!empty($request['token'])) return $request['token'];\r
+        //}\r
+        \r
+        //if($this->_request->isGet())\r
+        //{\r
+        //Try get\r
+            $request = $this->_request->get();\r
+            if(!empty($request['token'])) return $request['token'];\r
+        //}\r
+        \r
+        //no good, try the body\r
+        $body = json_decode($this->_request->getBody(), true);\r
+        if(!empty($body['token'])) return $body['token'];\r
+        \r
+        return null;\r
+    }\r
 }
\ No newline at end of file
index 76398f5..f09a537 100644 (file)
@@ -33,6 +33,10 @@ return [
     'Services\Routing\IRouter'                            => DI\object('Services\Routing\Router')\r
                                                                 ->constructor(DI\link('router.maps')),\r
     'Services\Uploads\IUploadManager'                     => DI\object('Services\Uploads\UploadManager'),\r
+    'Services\Uploads\IUploadQueueManager'                => DI\object('Services\Uploads\UploadQueueManager')\r
+                                                                ->lazy(),\r
+    'Services\ICacheUpdater'                              => DI\object('Services\CacheUpdater')\r
+                                                                ->lazy(),\r
     'Services\IUserSession'                               => DI\object('Services\UserSession'),\r
     'Services\IUserQuota'                                 => DI\object('Services\UserQuota'),\r
     'Services\Uploads\IFileFactory'                       => DI\object('Services\Uploads\FileFactory'),\r
index 7e3afb7..39fa2ff 100644 (file)
@@ -29,10 +29,17 @@ return [
         ]\r
     ],\r
     \r
-    '/simfiles/upload' => [\r
+    //'/simfiles/upload' => [\r
+    //    'controller' => 'Simfile',\r
+    //    'actions' => [\r
+    //        'POST' => 'upload'\r
+    //    ]\r
+    //],\r
+    \r
+    '/simfiles/upload/process' => [\r
         'controller' => 'Simfile',\r
         'actions' => [\r
-            'POST' => 'upload'\r
+            'GET' => 'processUploadQueue'\r
         ]\r
     ],\r
     \r