From 169ab35bee49eb5a0eec4ee002120dbdc524f579 Mon Sep 17 00:00:00 2001 From: Laurent Giovannoni <laurent.giovannoni@maarch.org> Date: Fri, 12 Jan 2018 10:46:21 +0100 Subject: [PATCH] FEAT #6490 add testU and MVC slim for convert --- core/Controllers/DocserverController.php | 2 +- core/Test/ResControllerTest.php | 8 + core/class/class_resource.php | 4 +- .../Controllers/ProcessConvertController.php | 44 +- .../Controllers/ProcessFulltextController.php | 433 +++++++----------- .../Models/ProcessConvertModelAbstract.php | 29 +- .../Models/ProcessFulltextModelAbstract.php | 215 +++++++++ modules/convert/Test/ProcessConvertTest.php | 103 ++++- modules/convert/Test/ProcessFulltextTest.php | 30 +- modules/convert/convert.sql | 40 +- phpunit.xml | 6 +- 11 files changed, 610 insertions(+), 304 deletions(-) diff --git a/core/Controllers/DocserverController.php b/core/Controllers/DocserverController.php index e404e7dcdf1..d8813ff39e4 100755 --- a/core/Controllers/DocserverController.php +++ b/core/Controllers/DocserverController.php @@ -70,7 +70,7 @@ class DocserverController ValidatorModel::stringType($aArgs, ['path']); if (!is_dir($aArgs['path'])) { - return ['errors' => '[createPathOnDocServer] Path does not exist']; + return ['errors' => '[createPathOnDocServer] Path does not exist ' . $aArgs['path']]; } error_reporting(0); diff --git a/core/Test/ResControllerTest.php b/core/Test/ResControllerTest.php index 9539dcac7bb..3ad8a0b5a38 100755 --- a/core/Test/ResControllerTest.php +++ b/core/Test/ResControllerTest.php @@ -63,6 +63,10 @@ class ResControllerTest extends TestCase $fileSource = 'test_source.txt'; + if (file_exists($path . $fileSource)) { + unlink($path . $fileSource); + } + $fp = fopen($path . $fileSource, 'a'); fwrite($fp, 'a unit test'); fclose($fp); @@ -163,6 +167,10 @@ class ResControllerTest extends TestCase $fileSource = 'test_source.txt'; + if (file_exists($path . $fileSource)) { + unlink($path . $fileSource); + } + $fp = fopen($path . $fileSource, 'a'); fwrite($fp, 'a unit test'); fclose($fp); diff --git a/core/class/class_resource.php b/core/class/class_resource.php index d1964c346cd..a0f8f743286 100755 --- a/core/class/class_resource.php +++ b/core/class/class_resource.php @@ -387,7 +387,9 @@ if($line->is_multi_docservers == "Y") { if ( $adrTable == 'adr_x' || - $adrTable == 'adr_attachments' + $adrTable == 'adr_letterbox' || + $adrTable == 'adr_attachments' || + $adrTable == 'adr_attachments_version' ) { $query = "select res_id, docserver_id, path, filename, offset_doc, fingerprint, adr_priority from " diff --git a/modules/convert/Controllers/ProcessConvertController.php b/modules/convert/Controllers/ProcessConvertController.php index 74eba8b0090..f5b63b707d4 100644 --- a/modules/convert/Controllers/ProcessConvertController.php +++ b/modules/convert/Controllers/ProcessConvertController.php @@ -25,6 +25,7 @@ namespace Convert\Controllers; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; +use Respect\Validation\Validator; use Convert\Models\ProcessConvertModel; use Core\Models\CoreConfigModel; use Core\Models\DocserverModel; @@ -44,6 +45,28 @@ class ProcessConvertController $this->libreOfficeExecutable = $libreOfficeExecutable; } + public function create(RequestInterface $request, ResponseInterface $response) + { + $data = $request->getParams(); + + $check = Validator::notEmpty()->validate($data['collId']); + $check = $check && Validator::stringType()->notEmpty()->validate($data['resTable']); + $check = $check && Validator::stringType()->notEmpty()->validate($data['adrTable']); + $check = $check && Validator::intType()->notEmpty()->validate($data['resId']); + $check = $check && Validator::stringType()->notEmpty()->validate($data['tmpDir']); + if (!$check) { + return $response->withStatus(400)->withJson(['errors' => 'Bad Request']); + } + + $return = ProcessConvertController::convert($data); + + if (empty($return) || !empty($return['errors'])) { + return $response->withStatus(500)->withJson(['errors' => '[ProcessConvertController create] ' . $return['errors']]); + } + + return $response->withJson($return); + } + /** * Ask for conversion * @@ -125,7 +148,9 @@ class ProcessConvertController 'value' => '', 'error' => 'file not exists : ' . $resourcePath, ); - ProcessConvertModel::manageErrorOnDb($resTable, $resId, '-1'); + ProcessConvertModel::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $returnArray; } //copy the resource on tmp directory @@ -136,7 +161,9 @@ class ProcessConvertController 'value' => '', 'error' => 'copy on tmp failed', ); - ProcessConvertModel::manageErrorOnDb($resTable, $resId, '-1'); + ProcessConvertModel::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $returnArray; } //now do the conversion ! @@ -157,7 +184,9 @@ class ProcessConvertController } if ($resultOfConversion['status'] <> '0') { - ProcessConvertModel::manageErrorOnDb($resTable, $resId, '-1'); + ProcessConvertModel::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $resultOfConversion; } //copy the result on docserver @@ -173,6 +202,7 @@ class ProcessConvertController 'docserverTypeId' => 'CONVERT' ]); + if (empty($storeResult)) { $returnArray = array( 'status' => '1', @@ -180,7 +210,9 @@ class ProcessConvertController 'error' => 'Ds of collection and ds type not found for convert:' . $collId . ' CONVERT', ); - ProcessConvertModel::manageErrorOnDb($resTable, $resId, '-1'); + ProcessConvertModel::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $returnArray; } @@ -209,7 +241,9 @@ class ProcessConvertController // LogsController::info(['message'=>var_export($resultCopyDs, true), 'code'=>7, ]); if ($resultOfUpDb['status'] <> '0') { - ProcessConvertModel::manageErrorOnDb($resTable, $resId, '-1'); + ProcessConvertModel::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $resultOfUpDb; } diff --git a/modules/convert/Controllers/ProcessFulltextController.php b/modules/convert/Controllers/ProcessFulltextController.php index 3e195129d1d..3d39152c58e 100644 --- a/modules/convert/Controllers/ProcessFulltextController.php +++ b/modules/convert/Controllers/ProcessFulltextController.php @@ -25,16 +25,15 @@ namespace Convert\Controllers; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; +use Respect\Validation\Validator; use Convert\Models\ProcessFulltextModel; +use Core\Models\CoreConfigModel; +use Core\Models\DocserverModel; use Core\Models\ResDocserverModel; +use Core\Models\ResModel; use Core\Controllers\LogsController; +use Core\Controllers\StoreController; -require_once 'core/class/class_functions.php'; -require_once 'core/class/class_db_pdo.php'; -// require_once 'core/class/class_db.php'; -// require_once 'core/docservers_tools.php'; -// require_once 'core/class/docservers_controler.php'; -// require_once 'core/services/ManageDocservers.php'; //include_once('html2text/html2text.php'); @@ -44,13 +43,14 @@ class ProcessFulltextController public function __construct($pdftotext = 'pdftotext') { - // Storing text in lucene index + // Storing text in lucpreene index set_include_path('apps/maarch_entreprise/tools/' . PATH_SEPARATOR . get_include_path() ); if(!@include('Zend/Search/Lucene.php')) { - set_include_path($GLOBALS['MaarchDirectory'].'apps/maarch_entreprise/tools/' + set_include_path($GLOBALS['MaarchDirectory'] + . 'apps/maarch_entreprise/tools/' . PATH_SEPARATOR . get_include_path() ); @@ -60,6 +60,41 @@ class ProcessFulltextController $this->pdftotext = $pdftotext; } + public function create(RequestInterface $request, ResponseInterface $response) + { + $data = $request->getParams(); + + $check = Validator::notEmpty()->validate($data['collId']); + $check = $check && Validator::stringType()->notEmpty()->validate($data['resTable']); + $check = $check && Validator::stringType()->notEmpty()->validate($data['adrTable']); + $check = $check && Validator::intType()->notEmpty()->validate($data['resId']); + $check = $check && Validator::stringType()->notEmpty()->validate($data['tmpDir']); + if (!$check) { + return $response->withStatus(400)->withJson(['errors' => 'Bad Request']); + } + + if ($data['createZendIndex']) { + $countColl = count($_SESSION['collections']); + for ($i=0;$i<$countColl;$i++) { + if ($_SESSION['collections'][$i]['id'] == 'letterbox_coll') { + $pathToLucene = $_SESSION['collections'][$i]['path_to_lucene_index']; + } + } + + $data['zendIndex'] = ProcessFulltextController::createZendIndexObject( + $pathToLucene + ); + } + + $return = ProcessFulltextController::fulltext($data); + + if (empty($return) || !empty($return['errors'])) { + return $response->withStatus(500)->withJson(['errors' => '[ProcessFulltextController create] ' . $return['errors']]); + } + + return $response->withJson($return); + } + /** * Ask for fulltext * @@ -122,7 +157,7 @@ class ProcessFulltextController $tmpDir = $args['tmpDir']; } - if(isset($args['path_to_lucene']) && !empty($args['path_to_lucene'])){ + if (isset($args['path_to_lucene']) && !empty($args['path_to_lucene'])) { $indexFileDirectory = $args['path_to_lucene']; } else { $countColl = count($_SESSION['collections']); @@ -133,35 +168,44 @@ class ProcessFulltextController } } } + if ($args['createZendIndex']) { + $countColl = count($_SESSION['collections']); + for ($i=0;$i<$countColl;$i++) { + if ($_SESSION['collections'][$i]['id'] == 'letterbox_coll') { + $pathToLucene = $_SESSION['collections'][$i]['path_to_lucene_index']; + } + } - $dbConv = new \Database($GLOBALS['configFile']); - - //retrieve path of the resource - $stmtConv = $dbConv->query("select * from " . $resTable - . " where res_id = ?", array($resId) - ); - $line = $stmtConv->fetchObject(); - - if ($line->res_id <> '') { + $args['zendIndex'] = ProcessFulltextController::createZendIndexObject( + $pathToLucene + ); + } + + $res = ResModel::getById(['resId' => $resId, 'resTable' => $args['resTable']]); + + if ($res['res_id'] <> '') { $resourcePath = ResDocserverModel::getSourceResourcePath( [ 'resTable' => $resTable, 'adrTable' => $adrTable, - 'resId' => $line->res_id, + 'resId' => $res['res_id'], 'adrType' => 'CONV' ] ); } if (!file_exists($resourcePath)) { $returnArray = array( - 'status' => '2', + 'status' => '1', 'value' => '', 'error' => 'file not already converted in pdf for fulltext. path :' . $resourcePath . ", adrType : CONV, adr_table : " . $adrTable, ); - $this->manageErrorOnDb($resTable, $resId, '-1'); + ProcessFulltextController::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $returnArray; } + //copy the resource on tmp directory $fileNameOnTmp = $tmpDir . rand() . rand(); if (!copy($resourcePath, $fileNameOnTmp)) { @@ -170,9 +214,12 @@ class ProcessFulltextController 'value' => '', 'error' => 'copy on tmp failed for fulltext. Copy ' . $resourcePath . ' to ' . $fileNameOnTmp, ); - $this->manageErrorOnDb($resTable, $resId, '-1'); + ProcessFulltextController::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $returnArray; } + //now do the fulltext ! if (!empty($args['zendIndex'])) { $resultOfConversion = $this->launchFulltext( @@ -190,9 +237,11 @@ class ProcessFulltextController $tmpDir ); } - + if ($resultOfConversion['status'] <> '0') { - $this->manageErrorOnDb($resTable, $resId, '-1'); + ProcessFulltextController::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); LogsController::executionTimeLog( $timestart, '', @@ -201,37 +250,54 @@ class ProcessFulltextController ); return $resultOfConversion; } - //find the target docserver - $targetDs = $ManageDocservers->findTargetDs($collId, 'FULLTEXT'); - if (empty($targetDs)) { + + //copy the result on docserver + // LogsController::info(['message'=>'avant cp ds', 'code'=>1112, ]); + $storeResult = StoreController::storeResourceOnDocServer([ + 'collId' => $collId, + 'fileInfos' => [ + 'tmpDir' => CoreConfigModel::getTmpPath(), + 'size' => filesize($fileNameOnTmp), + 'format' => 'TXT', + 'tmpFileName' => pathinfo($fileNameOnTmp, PATHINFO_FILENAME) . '.txt', + ], + 'docserverTypeId' => 'FULLTEXT' + ]); + + if (empty($storeResult)) { $returnArray = array( 'status' => '1', 'value' => '', 'error' => 'Ds of collection and ds type not found for fulltext:' . $collId . ' FULLTEXT', ); - $this->manageErrorOnDb($resTable, $resId, '-1'); + ProcessFulltextController::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $returnArray; } - //copy the result on docserver - $resultCopyDs = $ManageDocservers->copyResOnDS($fileNameOnTmp . '.txt', $targetDs); - if ($resultCopyDs['status'] <> '0') { - $this->manageErrorOnDb($resTable, $resId, '-1'); - return $resultCopyDs; - } - //update the database - $resultOfUpDb = $this->updateDatabase( - $collId, - $resTable, - $adrTable, - $resId, - $targetDs, - $resultCopyDs['value']['destinationDir'], - $resultCopyDs['value']['fileDestinationName'], - $args['zendIndex'] + + $targetDs = DocserverModel::getById(['id' => $storeResult['docserver_id']]); + + // LogsController::info(['message'=>'avant update', 'code'=>19, ]); + //update the Database + $resultOfUpDb = ProcessFulltextModel::updateDatabase( + [ + 'collId' => $collId, + 'resTable' => $resTable, + 'adrTable' => $adrTable, + 'resId' => $resId, + 'docserver' => $targetDs, + 'path' => $storeResult['destination_dir'], + 'fileName' => $storeResult['file_destination_name'], + 'zendIndex' => $args['zendIndex'] + ] ); + if ($resultOfUpDb['status'] <> '0') { - $this->manageErrorOnDb($resTable, $resId, '-1'); + ProcessFulltextModel::manageErrorOnDb( + ['resTable' => $resTable, 'resId' => $resId, 'result' => '-1'] + ); return $resultOfUpDb; } @@ -317,7 +383,7 @@ class ProcessFulltextController return $result; } - private function prepareIndexFullTextPdf($pathToFile, $tmpDir, $indexFileDirectory, $resId, $zendIndex) + private function prepareIndexFullTextPdf($pathToFile, $tmpDir, $indexFileDirectory, $resId, $zendIndex = "") { $timestart = microtime(true); if (is_file($pathToFile)) { @@ -326,7 +392,12 @@ class ProcessFulltextController $resultExtraction = exec("pdftotext " . escapeshellarg($pathToFile) . " " . escapeshellarg($tmpFile) ); - LogsController::executionTimeLog($timestart_fulltext, '', 'debug', '[TIMER] Convert_ProcessFulltextAbstract_Service::prepareIndexFullTextPdf__exec'); + LogsController::executionTimeLog( + $timestart_fulltext, + '', + 'debug', + '[TIMER] Convert_ProcessFulltextAbstract_Service::prepareIndexFullTextPdf__exec' + ); $fileContent = trim($this->readFileF($tmpFile)); @@ -363,23 +434,22 @@ class ProcessFulltextController public function createZendIndexObject($tempIndexFileDirectory, $numberOfIndexes = 1000) { //echo 'createZendIndexObject : ' . $numberOfIndexes . PHP_EOL; - $func = new functions(); $indexFileDirectory = (string) $tempIndexFileDirectory; // with version 1.12, we need a string, not an XML element if (!is_dir($indexFileDirectory)) { - $index = Zend_Search_Lucene::create($indexFileDirectory); + $index = \Zend_Search_Lucene::create($indexFileDirectory); } else { - if ($func->isDirEmpty($indexFileDirectory)) { - $index = Zend_Search_Lucene::create($indexFileDirectory); + if ($this->isDirEmpty($indexFileDirectory)) { + $index = \Zend_Search_Lucene::create($indexFileDirectory); } else { - $index = Zend_Search_Lucene::open($indexFileDirectory); + $index = \Zend_Search_Lucene::open($indexFileDirectory); } } - $index->setFormatVersion(Zend_Search_Lucene::FORMAT_2_3); + $index->setFormatVersion(\Zend_Search_Lucene::FORMAT_2_3); // we set the lucene format to 2.3 - Zend_Search_Lucene_Analysis_Analyzer::setDefault( - new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() + \Zend_Search_Lucene_Analysis_Analyzer::setDefault( + new \Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() ); //$index->MaxBufferedDocs(); @@ -388,6 +458,26 @@ class ProcessFulltextController return $index; } + /** + * Checks if a directory is empty + * + * @param $dir string The directory to check + * @return bool True if empty, False otherwise + */ + function isDirEmpty($dir) + { + $dir = opendir($dir); + $isEmpty = true; + while (($entry = readdir($dir)) !== false) { + if ($entry !== '.' && $entry !== '..' && $entry !== '.svn') { + $isEmpty = false; + break; + } + } + closedir($dir); + return $isEmpty; + } + /** * Commit the zend index at the end of the batch * @return nothing @@ -409,8 +499,7 @@ class ProcessFulltextController private function launchIndexFullText($fileContent, $tempIndexFileDirectory, $Id) { // $IndexFileDirectory is replace by tempIndexFileDirectory - $func = new functions(); - $fileContent = $func->normalize($fileContent); + $fileContent = \Core\Models\TextFormatModel::normalize(['string' => $fileContent]); $fileContent = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $fileContent); $indexFileDirectory = (string) $tempIndexFileDirectory; // with version 1.12, we need a string, not an XML element @@ -420,7 +509,7 @@ class ProcessFulltextController //$_ENV['logger']->write($indexFileDirectory . " not exists !", "ERROR", 2); $index = Zend_Search_Lucene::create($indexFileDirectory); } else { - if ($func->isDirEmpty($indexFileDirectory)) { + if ($this->isDirEmpty($indexFileDirectory)) { //$_ENV['logger']->write($indexFileDirectory . " empty !"); $index = Zend_Search_Lucene::create($indexFileDirectory); } else { @@ -430,23 +519,21 @@ class ProcessFulltextController $index->setFormatVersion(Zend_Search_Lucene::FORMAT_2_3); // we set the lucene format to 2.3 Zend_Search_Lucene_Analysis_Analyzer::setDefault( - new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() + new \Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() ); // we need utf8 for accents - $term = new Zend_Search_Lucene_Index_Term($Id, 'Id'); + $term = new \Zend_Search_Lucene_Index_Term($Id, 'Id'); foreach ($index->termDocs($term) as $id) { $index->delete($id); } //echo $fileContent; - $doc = new Zend_Search_Lucene_Document(); - $doc->addField(Zend_Search_Lucene_Field::UnIndexed('Id', (integer) $Id)); - $doc->addField(Zend_Search_Lucene_Field::UnStored( + $doc = new \Zend_Search_Lucene_Document(); + $doc->addField(\Zend_Search_Lucene_Field::UnIndexed('Id', (integer) $Id)); + $doc->addField(\Zend_Search_Lucene_Field::UnStored( 'contents', $fileContent) ); - //$func->show_array($doc); $index->addDocument($doc); $index->commit(); - //$func->show_array($index); //$index->optimize(); $result = 0; } else { @@ -467,8 +554,7 @@ class ProcessFulltextController { //echo 'launchIndexFullTextWithZendIndex' . PHP_EOL; // $IndexFileDirectory is replace by tempIndexFileDirectory - $func = new functions(); - $fileContent = $func->normalize($fileContent); + $fileContent = \Core\Models\TextFormatModel::normalize(['string' => $fileContent]); $fileContent = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $fileContent); // with version 1.12, we need a string, not an XML element @@ -476,14 +562,14 @@ class ProcessFulltextController if (strlen($fileContent) > 2) { try { // we need utf8 for accents - $term = new Zend_Search_Lucene_Index_Term($Id, 'Id'); + $term = new \Zend_Search_Lucene_Index_Term($Id, 'Id'); foreach ($index->termDocs($term) as $id) { $index->delete($id); } //echo $fileContent; - $doc = new Zend_Search_Lucene_Document(); - $doc->addField(Zend_Search_Lucene_Field::UnIndexed('Id', (integer) $Id)); - $doc->addField(Zend_Search_Lucene_Field::UnStored( + $doc = new \Zend_Search_Lucene_Document(); + $doc->addField(\Zend_Search_Lucene_Field::UnIndexed('Id', (integer) $Id)); + $doc->addField(\Zend_Search_Lucene_Field::UnStored( 'contents', $fileContent) ); //$func->show_array($doc); @@ -502,211 +588,9 @@ class ProcessFulltextController return $result; } - /** - * Updating the database with the location information of the document on the - * new docserver - * @param string $collId collection - * @param string $resTable res table - * @param string $adrTable adr table - * @param bigint $resId Id of the resource to process - * @param docserver $docserver docserver object - * @param string $path location of the resource on the docserver - * @param string $fileName file name of the resource on the docserver - * @return array $returnArray the result - */ - private function updateDatabase( - $collId, - $resTable, - $adrTable, - $resId, - $docserver, - $path, - $fileName, - $zendIndex = '' - ) { - try { - $docserver->path_template = str_replace( - DIRECTORY_SEPARATOR, - '#', - $docserver->path_template - ); - $path = str_replace($docserver->path_template, '', $path); - $dbConv = new \Database($GLOBALS['configFile']); - $query = "update convert_stack set status = 'P' where " - . " coll_id = ? and res_id = ?"; - $stmt = $dbConv->query( - $query, - array( - $collId, - $resId - ) - ); - - $query = "select * from " . $adrTable - . " where res_id = ? order by adr_priority"; - $stmt = $dbConv->query($query, array($resId)); - if ($stmt->rowCount() == 0) { - $query = "select docserver_id, path, filename, offset_doc, fingerprint" - . " from " . $resTable . " where res_id = ?"; - $stmt = $dbConv->query($query, array($resId)); - $recordset = $stmt->fetchObject(); - $resDocserverId = $recordset->docserver_id; - $resPath = $recordset->path; - $resFilename = $recordset->filename; - $resOffsetDoc = $recordset->offset_doc; - $fingerprintInit = $recordset->fingerprint; - $query = "select adr_priority_number from docservers " - . " where docserver_id = ?"; - $stmt = $dbConv->query($query, array($resDocserverId)); - $recordset = $stmt->fetchObject(); - $query = "insert into " . $adrTable . " (res_id, " - . "docserver_id, path, filename, offset_doc, fingerprint, " - . "adr_priority) values (?, ?, ?, ?, ?, ?, ?)"; - $stmt = $dbConv->query( - $query, - array( - $resId, - $resDocserverId, - $resPath, - $resFilename, - $resOffsetDoc, - $fingerprintInit, - $recordset->adr_priority_number - ) - ); - } - - $query = "select * from " . $adrTable - . " where res_id = ? and adr_type = 'TXT'"; - $stmt = $dbConv->query($query, array($resId)); - if ($stmt->rowCount() == 0) { - $query = "insert into " . $adrTable . " (res_id, docserver_id, " - . "path, filename, offset_doc, fingerprint, adr_priority, adr_type) values (" - . "?, ?, ?, ?, ?, ?, ?, ?)"; - $stmt = $dbConv->query( - $query, - array( - $resId, - $docserver->docserver_id, - $path, - $fileName, - $offsetDoc, - $fingerprint, - $docserver->adr_priority_number, - 'TXT' - ) - ); - } else { - $query = "update " . $adrTable . " set docserver_id = ?, " - . " path = ?, filename = ?, offset_doc = ?, fingerprint = ?, adr_priority = ?" - . " where res_id = ? and adr_type = ? "; - $stmt = $dbConv->query( - $query, - array( - $docserver->docserver_id, - $path, - $fileName, - $offsetDoc, - $fingerprint, - $docserver->adr_priority_number, - $resId, - 'TXT' - ) - ); - } - if ($_SESSION['user']['UserId'] <> '') { - $user = $_SESSION['user']['UserId']; - } else { - $user = 'CONVERT_BOT'; - } - $query = "insert into history (table_name, record_id, " - . "event_type, user_id, event_date, info, id_module) values (" - . "?, ?, 'ADD', '" . $user . "', " - . $dbConv->current_datetime() - . ", ?, 'convert')"; - $stmt = $dbConv->query( - $query, - array( - $resTable, - $resId, - "process fulltext done" - ) - ); - - $queryCpt = "select coalesce(custom_t15, '0') as custom_t15 from " . $resTable - . " where res_id = ?"; - $stmtCpt = $dbConv->query($queryCpt, array($resId)); - $rsCpt = $stmtCpt->fetchObject(); - $cptFullText = $rsCpt->custom_t15 + 1; - - if (!empty($zendIndex)) { - $query = "update " . $resTable - . " set fulltext_result = '1', is_multi_docservers = 'Y', custom_t15 = '" - . $cptFullText . "' where " - . " res_id = ?"; - } else { - $query = "update " . $resTable - . " set fulltext_result = '0', is_multi_docservers = 'Y', custom_t15 = '" - . $cptFullText . "' where " - . " res_id = ?"; - } - $stmt = $dbConv->query( - $query, - array( - $resId - ) - ); - $returnArray = array( - 'status' => '0', - 'value' => '', - 'error' => '', - ); - return $returnArray; - } catch (Exception $e) { - $returnArray = array( - 'status' => '1', - 'value' => '', - 'error' => $e->getMessage(), - ); - return $returnArray; - } - } - - /** - * Updating the database with the error code - * @param string $resTable res table - * @param bigint $resId Id of the resource to process - * @param string $result error code - * @return nothing - */ - private function manageErrorOnDb( - $resTable, - $resId, - $result - ) { - $dbConv = new \Database($GLOBALS['configFile']); - - $queryCpt = "select coalesce(custom_t15, '0') as custom_t15 from " . $resTable - . " where res_id = ?"; - $stmtCpt = $dbConv->query($queryCpt, array($resId)); - $rsCpt = $stmtCpt->fetchObject(); - $cptFullText = $rsCpt->custom_t15 + 1; - - $query = "update " . $resTable - . " set fulltext_result = ?, custom_t15 = '" . $cptFullText . "' where " - . " res_id = ?"; - $stmt = $dbConv->query( - $query, - array( - $result, - $resId - ) - ); - } - public static function optimizeLuceneIndex(array $args=[]){ $timestart = microtime(true); - // Prés-requis : + // prerequisites self::checkRequired($args, ['collId']); self::checkString($args, ['collId']); @@ -736,5 +620,4 @@ class ProcessFulltextController return true; } - } diff --git a/modules/convert/Models/ProcessConvertModelAbstract.php b/modules/convert/Models/ProcessConvertModelAbstract.php index 98a5d1ee3ad..bbfed61ef1c 100644 --- a/modules/convert/Models/ProcessConvertModelAbstract.php +++ b/modules/convert/Models/ProcessConvertModelAbstract.php @@ -155,20 +155,20 @@ class ProcessConvertModelAbstract ]); $queryCpt = DatabaseModel::select([ - 'select' => ["coalesce(custom_t9, '0') as custom_t9"], + 'select' => ["convert_attempts"], 'table' => [$aArgs['resTable']], 'where' => ['res_id = ?'], 'data' => [$aArgs['resId']], ]); - $cptConvert = $queryCpt[0]['custom_t9'] + 1; + $cptConvert = $queryCpt[0]['convert_attempts'] + 1; DatabaseModel::update([ 'table' => $aArgs['resTable'], 'set' => [ 'convert_result' => '1', 'is_multi_docservers' => 'Y', - 'custom_t9' => $cptConvert, + 'convert_attempts' => $cptConvert, ], 'where' => ['res_id = ?'], 'data' => [$aArgs['resId']] @@ -197,15 +197,26 @@ class ProcessConvertModelAbstract * @param string $result error code * @return nothing */ - public static function manageErrorOnDb( - $resTable, - $resId, - $result - ) { + public static function manageErrorOnDb(array $aArgs = []) + { + $attemptsRecord = DatabaseModel::select([ + 'select' => ['convert_attempts'], + 'table' => [$aArgs['resTable']], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']], + ]); + + if (empty($attemptsRecord)) { + $attempts = 0; + } else { + $attempts = $attemptsRecord[0]['convert_attempts'] + 1; + } + DatabaseModel::update([ 'table' => $aArgs['resTable'], 'set' => [ - 'convert_result' => $result, + 'convert_result' => $aArgs['result'], + 'convert_attempts' => $attempts, ], 'where' => ['res_id = ?'], 'data' => [$aArgs['resId']] diff --git a/modules/convert/Models/ProcessFulltextModelAbstract.php b/modules/convert/Models/ProcessFulltextModelAbstract.php index 61c3af97fb8..347566f823a 100644 --- a/modules/convert/Models/ProcessFulltextModelAbstract.php +++ b/modules/convert/Models/ProcessFulltextModelAbstract.php @@ -15,8 +15,190 @@ namespace Convert\Models; +use Core\Models\DatabaseModel; +use Core\Models\ValidatorModel; +use Core\Controllers\HistoryController; + class ProcessFulltextModelAbstract { + /** + * Updating the database with the location information of the document on the + * new docserver + * @param string $collId collection + * @param string $resTable res table + * @param string $adrTable adr table + * @param bigint $resId Id of the resource to process + * @param docserver $docserver docserver object + * @param string $path location of the resource on the docserver + * @param string $fileName file name of the resource on the docserver + * @param complex $zendIndex zend index object + * @return array $returnArray the result + */ + public static function updateDatabase(array $aArgs = []) + { + try { + ValidatorModel::notEmpty($aArgs, ['collId']); + ValidatorModel::notEmpty($aArgs, ['resTable']); + ValidatorModel::notEmpty($aArgs, ['adrTable']); + ValidatorModel::intVal($aArgs, ['resId']); + ValidatorModel::notEmpty($aArgs, ['docserver']); + ValidatorModel::notEmpty($aArgs, ['path']); + ValidatorModel::notEmpty($aArgs, ['fileName']); + + $aArgs['docserver']['path_template'] = str_replace( + DIRECTORY_SEPARATOR, + '#', + $aArgs['docserver']['path_template'] + ); + $aArgs['path'] = str_replace( + $aArgs['docserver']['path_template'], + '', + $aArgs['path'] + ); + + DatabaseModel::update([ + 'table' => 'convert_stack', + 'set' => [ + 'status' => 'P' + ], + 'where' => ['coll_id = ?', 'res_id = ?'], + 'data' => [$aArgs['collId'], $aArgs['resId']] + ]); + + $returnAdr = DatabaseModel::select([ + 'select' => ['*'], + 'table' => [$aArgs['adrTable']], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']], + 'order' => ['adr_priority'], + ]); + + if (empty($returnAdr)) { + $returnRes = DatabaseModel::select([ + 'select' => ['docserver_id, path, filename, offset_doc, fingerprint'], + 'table' => [$aArgs['resTable']], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']] + ]); + $returnRes = $returnRes[0]; + // LogsController::info(['message'=>$returnRes, 'code'=>8, ]); + $resDocserverId = $returnRes['docserver_id']; + $resPath = $returnRes['path']; + $resFilename = $returnRes['filename']; + $resOffsetDoc = $returnRes['offset_doc']; + $fingerprintInit = $returnRes['fingerprint']; + + $returnDs = DatabaseModel::select([ + 'select' => ['adr_priority_number'], + 'table' => ['docservers'], + 'where' => ['docserver_id = ?'], + 'data' => [$resDocserverId] + ]); + + DatabaseModel::insert([ + 'table' => $aArgs['adrTable'], + 'columnsValues' => [ + 'res_id' => $aArgs['resId'], + 'docserver_id' => $resDocserverId, + 'path' => $resPath, + 'filename' => $resFilename, + 'offset_doc' => $resOffsetDoc, + 'fingerprint' => $fingerprintInit, + 'adr_priority' => $returnDs[0]['adr_priority_number'], + ] + ]); + } + + $returnAdr = DatabaseModel::select([ + 'select' => ['*'], + 'table' => [$aArgs['adrTable']], + 'where' => ['res_id = ?', 'adr_type= ?'], + 'data' => [$aArgs['resId'], 'TXT'], + ]); + + if (empty($returnAdr)) { + DatabaseModel::insert([ + 'table' => $aArgs['adrTable'], + 'columnsValues' => [ + 'res_id' => $aArgs['resId'], + 'docserver_id' => $aArgs['docserver']['docserver_id'], + 'path' => $aArgs['path'], + 'filename' => $aArgs['fileName'], + 'offset_doc' => $offsetDoc, + 'fingerprint' => $fingerprint, + 'adr_priority' => $aArgs['docserver']['adr_priority_number'], + 'adr_type' => 'TXT', + ] + ]); + } else { + DatabaseModel::update([ + 'table' => $aArgs['adrTable'], + 'set' => [ + 'docserver_id' => $aArgs['docserver']['docserver_id'], + 'path' => $aArgs['path'], + 'filename' => $aArgs['fileName'], + 'offset_doc' => $offsetDoc, + 'fingerprint' => $fingerprint, + 'adr_priority' => $aArgs['docserver']['adr_priority_number'], + ], + 'where' => ['res_id = ?', "adr_type = ?"], + 'data' => [$aArgs['resId'], 'TXT'] + ]); + } + + HistoryController::add([ + 'tableName' => $aArgs['resTable'], + 'recordId' => (string) $aArgs['resId'], + 'eventType' => 'ADD', + 'info' => 'process fulltext done', + 'moduleId' => 'convert', + 'eventId' => 'fulltext', + ]); + + $queryCpt = DatabaseModel::select([ + 'select' => ["fulltext_attempts"], + 'table' => [$aArgs['resTable']], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']], + ]); + + $cptFulltext = $queryCpt[0]['fulltext_attempts'] + 1; + + if (!empty($aArgs['zendIndex'])) { + $fResult = 1; + } else { + $fResult = 0; + } + + DatabaseModel::update([ + 'table' => $aArgs['resTable'], + 'set' => [ + 'fulltext_result' => $fResult, + 'is_multi_docservers' => 'Y', + 'fulltext_attempts' => $cptFulltext, + ], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']] + ]); + + $returnArray = array( + 'status' => '0', + 'value' => '', + 'error' => '', + ); + + return $returnArray; + } catch (Exception $e) { + $returnArray = array( + 'status' => '1', + 'value' => '', + 'error' => $e->getMessage(), + ); + + return $returnArray; + } + } + public static function getById(array $aArgs = []) { ValidatorModel::notEmpty($aArgs, ['resId']); @@ -35,4 +217,37 @@ class ProcessFulltextModelAbstract return $aReturn[0]; } + + /** + * Updating the database with the error code + * @param string $resTable res table + * @param bigint $resId Id of the resource to process + * @param string $result error code + * @return nothing + */ + public static function manageErrorOnDb(array $aArgs = []) + { + $attemptsRecord = DatabaseModel::select([ + 'select' => ['fulltext_attempts'], + 'table' => [$aArgs['resTable']], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']], + ]); + + if (empty($attemptsRecord)) { + $attempts = 0; + } else { + $attempts = $attemptsRecord[0]['fulltext_attempts'] + 1; + } + + DatabaseModel::update([ + 'table' => $aArgs['resTable'], + 'set' => [ + 'fulltext_result' => $aArgs['result'], + 'fulltext_attempts' => $attempts, + ], + 'where' => ['res_id = ?'], + 'data' => [$aArgs['resId']] + ]); + } } diff --git a/modules/convert/Test/ProcessConvertTest.php b/modules/convert/Test/ProcessConvertTest.php index c1f69f11bae..c25c0dcc3e3 100644 --- a/modules/convert/Test/ProcessConvertTest.php +++ b/modules/convert/Test/ProcessConvertTest.php @@ -9,24 +9,119 @@ namespace MaarchTest; use PHPUnit\Framework\TestCase; -use MaarchTest\DocserverControllerTest; class ProcessConvertTest extends TestCase { public function testconvert () { + $action = new \Core\Controllers\ResController(); + + $environment = \Slim\Http\Environment::mock( + [ + 'REQUEST_METHOD' => 'POST', + ] + ); + + $path = $_SESSION['config']['tmppath'] . '/test/'; + + if (!is_dir($path)) { + mkdir($path); + } + + $fileSource = 'test_source.txt'; + + if (file_exists($path . $fileSource)) { + unlink($path . $fileSource); + } + + $fp = fopen($path . $fileSource, 'a'); + fwrite($fp, 'a unit test for PHP CONVERSION lorem ipsum...'); + fclose($fp); + + $fileContent = file_get_contents($path . $fileSource, FILE_BINARY); + $encodedFile = base64_encode($fileContent); + //echo $encodedFile . PHP_EOL;exit; + + $data = []; + + array_push( + $data, + array( + 'column' => 'subject', + 'value' => 'UNIT TEST from slim', + 'type' => 'string', + ) + ); + + array_push( + $data, + array( + 'column' => 'type_id', + 'value' => 110, + 'type' => 'integer', + ) + ); + + array_push( + $data, + array( + 'column' => 'custom_t1', + 'value' => 'TEST', + 'type' => 'string', + ) + ); + + array_push( + $data, + array( + 'column' => 'custom_t10', + 'value' => 'lgi@maarch.org', + 'type' => 'string', + ) + ); + + $aArgs = [ + 'encodedFile' => $encodedFile, + 'data' => $data, + 'collId' => 'letterbox_coll', + 'table' => 'res_letterbox', + 'fileFormat' => 'txt', + 'status' => 'new', + ]; + + $request = \Slim\Http\Request::createFromEnvironment($environment); + $response = new \Slim\Http\Response(); + $fullRequest = \httpRequestCustom::addContentInBody($aArgs, $request); + + $response = $action->create($fullRequest, $response); + $responseBody = json_decode((string)$response->getBody()); + + $resId = $responseBody->resId; + + if (!defined("_RES_ID_TEST_CONVERT")) { + define("_RES_ID_TEST_CONVERT", $resId); + } + + //real test + $request = \Slim\Http\Request::createFromEnvironment($environment); $action = new \Convert\Controllers\ProcessConvertController(); $aArgs = [ 'collId' => 'letterbox_coll', 'resTable' => 'res_letterbox', 'adrTable' => 'adr_letterbox', - 'resId' => 137, + 'resId' => _RES_ID_TEST_CONVERT, 'tmpDir' => $_SESSION['config']['tmppath'] ]; - $response = $action->convert($aArgs); + $response = new \Slim\Http\Response(); + $fullRequest = \httpRequestCustom::addContentInBody($aArgs, $request); + + $response = $action->create($fullRequest, $response); + //var_dump($response); + $responseBody = json_decode((string)$response->getBody()); + $status = $responseBody->status; - $this->assertArrayHasKey('status', $response); + $this->assertEquals('0', $status); } } diff --git a/modules/convert/Test/ProcessFulltextTest.php b/modules/convert/Test/ProcessFulltextTest.php index c21f8022abf..fa01d225016 100644 --- a/modules/convert/Test/ProcessFulltextTest.php +++ b/modules/convert/Test/ProcessFulltextTest.php @@ -9,25 +9,45 @@ namespace MaarchTest; use PHPUnit\Framework\TestCase; -use MaarchTest\DocserverControllerTest; class ProcessFulltextTest extends TestCase { public function testfulltext () { + + if (!defined("_RES_ID_TEST_CONVERT")) { + define("_RES_ID_TEST_CONVERT", 100); + } + $action = new \Convert\Controllers\ProcessFulltextController(); + $environment = \Slim\Http\Environment::mock( + [ + 'REQUEST_METHOD' => 'POST', + ] + ); + + $request = \Slim\Http\Request::createFromEnvironment($environment); + $aArgs = [ 'collId' => 'letterbox_coll', 'resTable' => 'res_letterbox', 'adrTable' => 'adr_letterbox', - 'resId' => 100, - 'tmpDir' => $_SESSION['config']['tmppath'] + 'resId' => _RES_ID_TEST_CONVERT, + 'tmpDir' => $_SESSION['config']['tmppath'], + 'createZendIndex' => true ]; - $response = $action->fulltext($aArgs); + $response = new \Slim\Http\Response(); + $fullRequest = \httpRequestCustom::addContentInBody($aArgs, $request); - $this->assertArrayHasKey('status', $response); + $response = $action->create($fullRequest, $response); + //var_dump($response); + $responseBody = json_decode((string)$response->getBody()); + //var_dump($responseBody); + $status = $responseBody->status; + + $this->assertEquals('0', $status); } } diff --git a/modules/convert/convert.sql b/modules/convert/convert.sql index a0a088bf3aa..d06cc55d8e0 100644 --- a/modules/convert/convert.sql +++ b/modules/convert/convert.sql @@ -17,6 +17,38 @@ ALTER TABLE res_version_attachments DROP COLUMN IF EXISTS convert_result; ALTER TABLE res_version_attachments ADD COLUMN convert_result character varying(10) DEFAULT NULL::character varying; +--convert attempts +ALTER TABLE res_attachments DROP COLUMN IF EXISTS convert_attempts; +ALTER TABLE res_attachments ADD COLUMN convert_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_letterbox DROP COLUMN IF EXISTS convert_attempts; +ALTER TABLE res_letterbox ADD COLUMN convert_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_x DROP COLUMN IF EXISTS convert_attempts; +ALTER TABLE res_x ADD COLUMN convert_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_version_attachments DROP COLUMN IF EXISTS convert_attempts; +ALTER TABLE res_version_attachments ADD COLUMN convert_attempts integer DEFAULT NULL::integer; + +--fulltext attempts +ALTER TABLE res_attachments DROP COLUMN IF EXISTS fulltext_attempts; +ALTER TABLE res_attachments ADD COLUMN fulltext_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_letterbox DROP COLUMN IF EXISTS fulltext_attempts; +ALTER TABLE res_letterbox ADD COLUMN fulltext_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_x DROP COLUMN IF EXISTS fulltext_attempts; +ALTER TABLE res_x ADD COLUMN fulltext_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_version_attachments DROP COLUMN IF EXISTS fulltext_attempts; +ALTER TABLE res_version_attachments ADD COLUMN fulltext_attempts integer DEFAULT NULL::integer; + +--tnl attempts +ALTER TABLE res_attachments DROP COLUMN IF EXISTS tnl_attempts; +ALTER TABLE res_attachments ADD COLUMN tnl_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_letterbox DROP COLUMN IF EXISTS tnl_attempts; +ALTER TABLE res_letterbox ADD COLUMN tnl_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_x DROP COLUMN IF EXISTS tnl_attempts; +ALTER TABLE res_x ADD COLUMN tnl_attempts integer DEFAULT NULL::integer; +ALTER TABLE res_version_attachments DROP COLUMN IF EXISTS tnl_attempts; +ALTER TABLE res_version_attachments ADD COLUMN tnl_attempts integer DEFAULT NULL::integer; + + + --thumbnails result ALTER TABLE res_attachments DROP COLUMN IF EXISTS tnl_result; ALTER TABLE res_attachments ADD COLUMN tnl_result character varying(10) DEFAULT NULL::character varying; @@ -99,9 +131,13 @@ WITH (OIDS=FALSE); UPDATE docservers set docserver_type_id = 'DOC' where docserver_type_id <> 'TEMPLATES' and docserver_type_id <> 'TNL'; +DELETE FROM docserver_types where docserver_type_id = 'DOC'; +INSERT INTO docserver_types (docserver_type_id, docserver_type_label, enabled, is_container, container_max_number, is_compressed, compression_mode, is_meta, meta_template, is_logged, log_template, is_signed, fingerprint_mode) +VALUES ('DOC', 'Documents', 'Y', 'N', 0, 'N', 'NONE', 'N', 'NONE', 'N', 'NONE', 'Y', 'SHA512'); + DELETE FROM docserver_types where docserver_type_id = 'CONVERT'; INSERT INTO docserver_types (docserver_type_id, docserver_type_label, enabled, is_container, container_max_number, is_compressed, compression_mode, is_meta, meta_template, is_logged, log_template, is_signed, fingerprint_mode) -VALUES ('CONVERT', 'Conversions', 'Y', 'N', 0, 'N', 'NONE', 'N', 'NONE', 'N', 'NONE', 'Y', 'NONE'); +VALUES ('CONVERT', 'Conversions', 'Y', 'N', 0, 'N', 'NONE', 'N', 'NONE', 'N', 'NONE', 'Y', 'SHA256'); DELETE FROM docservers where docserver_id = 'CONVERT_MLB'; INSERT INTO docservers (docserver_id, docserver_type_id, device_label, is_readonly, enabled, size_limit_number, actual_size_number, path_template, ext_docserver_info, chain_before, chain_after, creation_date, closing_date, coll_id, priority_number, docserver_location_id, adr_priority_number) @@ -136,7 +172,7 @@ update docservers set docserver_id = 'TNL_MLB', priority_number = 12 where docse DELETE FROM docserver_types where docserver_type_id = 'FULLTEXT'; INSERT INTO docserver_types (docserver_type_id, docserver_type_label, enabled, is_container, container_max_number, is_compressed, compression_mode, is_meta, meta_template, is_logged, log_template, is_signed, fingerprint_mode) -VALUES ('FULLTEXT', 'FULLTEXT', 'Y', 'N', 0, 'N', 'NONE', 'N', 'NONE', 'N', 'NONE', 'Y', 'NONE'); +VALUES ('FULLTEXT', 'FULLTEXT', 'Y', 'N', 0, 'N', 'NONE', 'N', 'NONE', 'N', 'NONE', 'Y', 'SHA256'); DELETE FROM docservers where docserver_id = 'FULLTEXT_MLB'; INSERT INTO docservers (docserver_id, docserver_type_id, device_label, is_readonly, enabled, size_limit_number, actual_size_number, path_template, ext_docserver_info, chain_before, chain_after, creation_date, closing_date, coll_id, priority_number, docserver_location_id, adr_priority_number) diff --git a/phpunit.xml b/phpunit.xml index 503790f2f46..08ec051396b 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -10,8 +10,10 @@ <file>core/Test/StatusControllerTest.php</file> <file>core/Test/UserControllerTest.php</file> <file>core/Test/BasketControllerTest.php</file> - <!--file>modules/convert/Test/ProcessConvertTest.php</file--> - <!--file>modules/convert/Test/ProcessFulltextTest.php</file--> + <!-- <file>modules/convert/Test/ProcessConvertTest.php</file> + <file>modules/convert/Test/ProcessFulltextTest.php</file> + <file>modules/convert/Test/ProcessThumbnailsTest.php</file> + <file>modules/convert/Test/ProcessManageConvertTest.php</file> --> </testsuite> </testsuites> <filter> -- GitLab