Skip to content
GitLab
Commit 260561ef authored by Arnaud Pauget's avatar Arnaud Pauget
Browse files

Merge branch 'fix/20585_Medona_Transfer_large_file_upload' into 'develop' 

fix/20585_Medona_Transfer_large_file_upload

See merge request !729
parents e16b6ac9 b98d5b91
Pipeline #19087 failed with stages
in 38 seconds
Hide whitespace changes
Inline Side-by-side
core/Kernel/PresentationKernel.php
View file @ 260561ef
......@@ -154,8 +154,7 @@ class PresentationKernel
if (!is_null($this->request->body)) {
switch ($this->request->contentType) {
case 'url':
$contents = stream_get_contents($this->request->body);
$bodyArguments = \core\Encoding\url::decode($contents);
$bodyArguments = $this->request->body;
break;
case 'json':
......
core/Kernel/ServiceKernel.php
View file @ 260561ef
......@@ -200,8 +200,7 @@ class ServiceKernel extends AbstractKernel
break;
case 'url':
$contents = stream_get_contents($this->request->body);
$bodyArguments = \core\Encoding\url::decode($contents);
$bodyArguments = $this->request->body;
break;
case 'json':
......
core/Request/HttpRequest.php
View file @ 260561ef
......@@ -52,14 +52,7 @@ class HttpRequest
$this->getAuthentication();
$this->body = fopen('php://temp', 'w+');
$input = fopen('php://input', 'r');
$length = stream_copy_to_stream($input, $this->body);
rewind($this->body);
if ($length == 0) {
$this->body = null;
}
$this->getBody();
$this->query = urldecode($_SERVER['QUERY_STRING']);
......@@ -177,13 +170,76 @@ class HttpRequest
return $this->queryType;
}
protected function parseUrlBody()
protected function getBody()
{
if ($this->contentType == 'url') {
$this->body = array_merge_recursive($_POST, $this->getFiles());
} else {
$this->body = fopen('php://temp', 'w+');
$input = fopen('php://input', 'r');
$length = stream_copy_to_stream($input, $this->body);
rewind($this->body);
if ($length == 0) {
$this->body = null;
}
}
}
protected function getFiles()
{
if ($this->type == 'url' && !empty($this->body)) {
$bodyArguments = array();
\parse_str($this->body, $bodyArguments);
$this->arguments = array_merge($bodyArguments, $this->arguments);
$files = [];
$keys = ['name', 'type', 'tmp_name', 'error', 'size'];
foreach ($_FILES as $name => $item) {
// Standard form-data files at root level
if (is_string($item['name'])) {
$file = $item;
if ($file['error'] == 0) {
$file['handler'] = fopen($file['tmp_name'], 'r');
}
$files[$name] = $file;
continue;
}
// Merged array of file values
// root-level name =>
// 'name' => array of subfile names
// 'type' => array of subfile types
// ...
foreach ($item['name'] as $fname => $value) {
// 2nd level files
if (is_string($value)) {
$file = [];
foreach ($keys as $key) {
$file[$key] = $item[$key][$fname];
}
if ($file['error'] == 0) {
$file['handler'] = fopen($file['tmp_name'], 'r');
}
$files[$name][$fname] = $file;
continue;
}
// 3rd level : multiple files with one name
if (is_array($value)) {
foreach ($value as $index => $svalue) {
$file = [];
foreach ($keys as $key) {
$file[$key] = $item[$key][$fname][$index];
}
if ($file['error'] == 0) {
$file['handler'] = fopen($file['tmp_name'], 'r');
}
$files[$name][$fname][$index] = $file;
}
}
}
}
return $files;
}
protected function parseUrl()
......
data/maarchRM/conf/vhost.conf.default
View file @ 260561ef
......@@ -38,7 +38,7 @@
SetEnv LAABS_LOG "../data/maarchRM/log.txt"
SetEnv LAABS_TMP_DIR "../data/maarchRM/tmp"
SetEnv LAABS_CONTENT_TYPES "url:application/x-www-form-urlencoded;html:text/html,application/xhtml+xml;xml:application/xml;json:application/json,application/javascript;soap:application/soap+xml;csv:text/csv"
SetEnv LAABS_CONTENT_TYPES "url:application/x-www-form-urlencoded,multipart/form-data;html:text/html,application/xhtml+xml;xml:application/xml;json:application/json,application/javascript;soap:application/soap+xml;csv:text/csv"
#SetEnv LAABS_CONTENT_LANGUAGES "fr:fr,fr-fr,fr-ca"
SetEnv LAABS_CACHE_CONTROL "public, max-age=3600"
......
data/maarchRM/conf/vhost.win.conf.default
View file @ 260561ef
......@@ -38,7 +38,7 @@
SetEnv LAABS_TMP_DIR "C:/TEMP"
# SetEnv LAABS_PHP_INI "php.ini"
SetEnv LAABS_CONTENT_TYPES "url:application/x-www-form-urlencoded;html:text/html,application/xhtml+xml;xml:application/xml;json:application/json,application/javascript;soap:application/soap+xml;csv:text/csv"
SetEnv LAABS_CONTENT_TYPES "url:application/x-www-form-urlencoded,multipart/formdata;html:text/html,application/xhtml+xml;xml:application/xml;json:application/json,application/javascript;soap:application/soap+xml;csv:text/csv"
SetEnv LAABS_CONTENT_LANGUAGES "fr:fr,fr-fr,fr-ca"
SetEnv LAABS_CACHE_CONTROL "public, max-age=3600"
SetEnv LAABS_BUFFER_MODE 2
......
src/bundle/medona/Connectors/Multiparts.php
View file @ 260561ef
......@@ -50,10 +50,6 @@ class Multiparts
*/
public function receive($package, $params, $messageDirectory)
{
if (!isset($package->data) || empty($package->data)) {
throw new \core\Exception\BadRequestException("Package data is mandatory", 400);
}
if (isset($messageDirectory) && !empty($messageDirectory)) {
if (!is_dir($messageDirectory)) {
throw new \core\Exception\BadRequestException("MessageDirectory is not a directory", 400);
......@@ -61,14 +57,18 @@ class Multiparts
$this->messageDirectory = $messageDirectory;
}
if (is_resource($package->data)) {
$data = stream_get_contents(\core\Encoding\Base64::decode($package->data));
} elseif (filter_var($package->data, FILTER_VALIDATE_URL)) {
// TODO verify
} elseif (preg_match('%^[a-zA-Z0-9\\\\/+]*={0,2}$%', $package->data)) {
$data = \core\Encoding\Base64::decode($package->data);
} elseif (is_file($package->data)) {
$data = file_get_contents($package->data);
if (is_object($package)) {
if (is_resource($package->data)) {
$data = stream_get_contents(\core\Encoding\Base64::decode($package));
} elseif (filter_var($package->data, FILTER_VALIDATE_URL)) {
// TODO verify
} elseif (preg_match('%^[a-zA-Z0-9\\\\/+]*={0,2}$%', $package->data)) {
$data = \core\Encoding\Base64::decode($package->data);
} elseif (is_file($package->data)) {
$data = file_get_contents($package->data);
}
} elseif(is_array($package)) {
$data = stream_get_contents($package['handler']);
}
// to use to modify xml, replace namespace for seda2
......@@ -88,45 +88,50 @@ class Multiparts
if (!empty($params['attachments'])) {
foreach ($params['attachments'] as $key => $attachment) {
if (empty($attachment->name)) {
throw new \core\Exception\BadRequestException("Attachment name is mandatory", 400);
}
$attachmentFileName = $messageDirectory . DIRECTORY_SEPARATOR . $attachment->name;
if ($attachment->encoding == 'base64') {
switch (true) {
case is_resource($attachment->data):
case is_string($attachment->data) &&
(
filter_var(substr($attachment->data, 0, 10), FILTER_VALIDATE_URL) ||
is_file($attachment->data)
):
$handler = \core\Encoding\Base64::decode($attachment->data);
file_put_contents($attachmentFileName, stream_get_contents($handler));
break;
case is_string($attachment->data):
file_put_contents($attachmentFileName, \core\Encoding\Base64::decode($attachment->data));
break;
default:
throw new \core\Exception\BadRequestException("Data attachment format is not valid", 400);
break;
if (is_object($attachment)) {
if (empty($attachment->name)) {
throw new \core\Exception\BadRequestException("Attachment name is mandatory", 400);
}
} else {
switch (true) {
case is_resource($attachment->data):
case is_string($attachment->data) &&
(
filter_var(substr($attachment->data, 0, 10), FILTER_VALIDATE_URL) ||
is_file($attachment->data)
):
file_put_contents($attachmentFileName, file_get_contents($attachment->data));
break;
case is_string($attachment->data):
file_put_contents($attachmentFileName, $attachment->data);
break;
default:
throw new \core\Exception\BadRequestException("Data attachment format is not valid", 400);
break;
$attachmentFileName = $messageDirectory . DIRECTORY_SEPARATOR . $attachment->name;
if ($attachment->encoding == 'base64') {
switch (true) {
case is_resource($attachment->data):
case is_string($attachment->data) &&
(
filter_var(substr($attachment->data, 0, 10), FILTER_VALIDATE_URL) ||
is_file($attachment->data)
):
$handler = \core\Encoding\Base64::decode($attachment->data);
file_put_contents($attachmentFileName, stream_get_contents($handler));
break;
case is_string($attachment->data):
file_put_contents($attachmentFileName, \core\Encoding\Base64::decode($attachment->data));
break;
default:
throw new \core\Exception\BadRequestException("Data attachment format is not valid", 400);
break;
}
} else {
switch (true) {
case is_resource($attachment->data):
case is_string($attachment->data) &&
(
filter_var(substr($attachment->data, 0, 10), FILTER_VALIDATE_URL) ||
is_file($attachment->data)
):
file_put_contents($attachmentFileName, file_get_contents($attachment->data));
break;
case is_string($attachment->data):
file_put_contents($attachmentFileName, $attachment->data);
break;
default:
throw new \core\Exception\BadRequestException("Data attachment format is not valid", 400);
break;
}
}
} elseif (is_array($attachment)) {
$attachmentFileName = $messageDirectory . DIRECTORY_SEPARATOR . $attachment['name'];
file_put_contents($attachmentFileName, stream_get_contents($attachment['handler']));
}
$this->attachments[] = $attachmentFileName;
......
src/bundle/medona/Connectors/Zip.php
View file @ 260561ef
......@@ -26,19 +26,6 @@ class Zip
{
protected $messageDirectory;
/**
*
*
* @param string $messageDirectory [description]
*/
public function __construct($messageDirectory)
{
$this->messageDirectory = \laabs::configuration("medona")['messageDirectory'];
if (!is_dir($this->messageDirectory)) {
mkdir($this->messageDirectory, 0777, true);
}
}
/**
* Get archive transfer transformed by connector
*
......@@ -49,21 +36,26 @@ class Zip
*/
public function receive($package, $params, $messageDirectory)
{
if (!isset($package->data) || empty($package->data)) {
if (is_object($package) && (!isset($package->data) || empty($package->data))) {
throw new \core\Exception\BadRequestException("Package data is mandatory", 400);
}
if (isset($messageDirectory) && !empty($messageDirectory)) {
if (!is_dir($messageDirectory)) {
throw new \core\Exception\BadRequestException("MessageDirectory is not a directory", 400);
}
$this->messageDirectory = $messageDirectory;
if (!is_dir($messageDirectory)) {
throw new \core\Exception\BadRequestException("MessageDirectory is not a directory", 400);
}
$this->messageDirectory = $messageDirectory;
if (is_object($package)) {
$data = $package->data;
$encoding = $package->encoding;
} elseif (is_array($package)) {
$data = $package['handler'];
$encoding = null;
}
$packageName = $package->name ?? (string) \laabs::newId();
$zipTmpFile = \laabs\tempnam();
$this->receiveFile($package->data, $zipTmpFile, $package->encoding);
$this->receiveFile($data, $zipTmpFile, $encoding);
$tmpDir = \laabs\tempdir();
......@@ -108,6 +100,9 @@ class Zip
} else {
switch (true) {
case is_resource($data):
file_put_contents($zipTmpFile, stream_get_contents($data));
break;
case is_string($data) &&
(
filter_var(substr($data, 0, 10), FILTER_VALIDATE_URL) ||
......
src/bundle/medona/Controller/ArchiveTransfer.php
View file @ 260561ef
......@@ -71,7 +71,9 @@ class ArchiveTransfer extends abstractMessage
*/
public function receiveSource($package, $connector, $params = [])
{
if (!isset($this->packageConnectors[$connector]) || empty($this->packageConnectors[$connector])) {
if (!isset($this->packageConnectors[$connector])
|| empty($this->packageConnectors[$connector])
|| !isset($this->packageConnectors[$connector]['service'])) {
throw \laabs::newException('medona/invalidMessageException', "Invalid message: unknown connector", 400);
}
......@@ -87,19 +89,10 @@ class ArchiveTransfer extends abstractMessage
$params = $this->checkParamsConstraints($confParams, $params);
$message = $this->createNewMessage($schema);
$messageDirectory = $this->messageDirectory.DIRECTORY_SEPARATOR.(string) $message->messageId;
if (isset($connectorConf['service'])) {
$connectorService = \laabs::newService($connectorConf['service']);
$message->path = $connectorService->receive(
$package,
$params,
$this->messageDirectory.DIRECTORY_SEPARATOR.(string) $message->messageId
);
} else {
$messageFile = $package;
$attachments = [];
}
$connectorService = \laabs::newService($connectorConf['service']);
$message->path = $connectorService->receive($package, $params, $messageDirectory);
// Traiter le schéma spécifique
$this->receiveMessage($message);
......
src/presentation/maarchRM/Resources/view/medona/archiveTransfer/messageImport.html
View file @ 260561ef
......@@ -81,7 +81,7 @@
<div class="col-md-6">
</div>
<span class="hide" id="byte_text">byte</span>
<span class="hide" id="byte_text">bytes</span>
<span class="hide" id="import_text">Import</span>
<span class="hide" id="validation_text">Validation</span>
<span class="hide" id="remove_text">Remove</span>
......@@ -136,7 +136,7 @@ var zipContent = "";
e.preventDefault();
e.stopPropagation();
$('#packageDropZone').css('opacity', '0.5');
uploadMessage(e.originalEvent.dataTransfer.files[0]);
selectPackageFile();
}
}
}
......@@ -149,7 +149,7 @@ var zipContent = "";
$('#packagefileBrowser').on('change', function() {
uploadMessage($(this).get(0).files[0]);
selectPackageFile();
})
// Cancel
......@@ -162,122 +162,57 @@ var zipContent = "";
$('#userInputs').empty();
})
$('#messageImport_form').on('submit', function(e) {
e.preventDefault();
});
// Submit
$('#messageImport_submit').on('click', function() {
submit();
});
/* ------ FUNCTION ------ */
function isMissingRequiredParams() {
if ($('input[required]').filter(function() { return $(this).val() == ""; }).length != 0) {
return true;
}
return false;
}
function selectPackageFile()
{
let file = $('#packagefileBrowser').get(0).files[0];
$('#packageFileName').html(file.name);
$('#packageFileSize').html(file.size + ' ' + $('#byte_text').html());
$('#packageFileType').html(file.type);
$('#packageDropZone').addClass('hide');
$('#packageFileInfo').removeClass('hide');
}
$('#messageImport_form').on('submit', function(e) {
e.preventDefault();
});
// Submit
$('#messageImport_submit').on('click', function() {
function submit() {
if (isMissingRequiredParams()) {
return;
}
let source = $("#messageImport_source").val();
if (source != "" && source !== null) {
var url = '/transfer/source';
var data = JSON.stringify(serializeSource())
var contentType = 'application/json';
} else {
if (source == "" && source == null) {
gritter.show($('#empty_connectory_text').html(), false);
return;
}
$(this).attr('disabled',true);
$(this).attr('disabled', true);
$('#messageImport_cancel').attr('disabled', true);
ajax($('#messageImport_submit'), {
type : 'POST',
url : url,
data : data,
contentType : contentType,
dataType : 'json',
success : function (response) {
if (response.status != false) {
gritter.show(response.message, response.status, response.errors);
validateMessage(response.messageId);
} else {
gritter.show(response.message, response.status);
var arg = "";
$.each(response.errors,function (key,error) {
arg += "<br/>";
if (error.variables != null && error.variables.property != null) {
arg += error.variables.property + ': ';
}
arg += error.message;
});
$('#messageImport_errorTitle').html(response.message);
$('#messageImport_errorDetail').html(arg);
$('#messageImport_erreur').removeClass('hide');
$('#messageImport_submit').attr('disabled',false);
$('#messageImport_cancel').attr('disabled',false);
}
},
error : function (response) {
gritter.show(response.responseJSON.message, response.responseJSON.status);
var arg = "";
$.each(response.responseJSON.errors,function (key,error) {
arg += "<br/>";
if (error.variables != null && error.variables.property != null) {
arg += error.variables.property + ': ';
}
arg += error.message;
});
$('#messageImport_errorTitle').text(response.responseJSON.message);
$('#messageImport_errorDetail').html(arg);
$('#messageImport_erreur').removeClass('hide');
$('#messageImport_cancel').click();
$('#messageImport_submit').attr('disabled',false);
$('#messageImport_cancel').attr('disabled',false);
}
});
})
/* ------ FUNCTION ------ */
function uploadMessage(file) {
oFileReader = new FileReader();
if ( file.type == "application/x-zip-compressed" || file.type == "application/zip") {
oFileReader.readAsDataURL(file);
} else {
oFileReader.readAsText(file);
}
oFileReader.onload = function() {
if ( file.type == "application/x-zip-compressed" || file.type == "application/zip") {
$('#packagefileBrowser').data('base64', oFileReader.result.replace(/^data:.*?;base64,/, ""));
$('#packagefileBrowser').data('filename', file.name);
$('#packagefileBrowser').data('size', file.size);
} else {
$('#packagefileBrowser').data('base64', btoa(unescape(encodeURIComponent(oFileReader.result))));
$('#packagefileBrowser').data('filename', file.name);
$('#packagefileBrowser').data('size', file.size);
$('#packagefileBrowser').data('type', file.type);
}
let file = $('#packagefileBrowser').get(0).files[0];
$('#packageFileName').html(file.name);
$('#packageFileSize').html(file.size + ' ' + $('#byte_text').html());
var formData = new FormData();
formData.append("package", file);
formData.append("connector", source);
if (!file.type || file.type == "") {
gritter.show($('#empty_file_type_text').html(), false);
$('#messageImport_submit').attr('disabled',true);
} else {
$('#packageFileType').html(file.type);
}
$('#packageDropZone').addClass('hide');
$('#packageFileInfo').removeClass('hide');
};
serializeParams(formData);
sendPackage(formData);
}
function validateMessage(messageId) {
......@@ -336,103 +271,51 @@ var zipContent = "";
});
}
function serialize() {
var jsonObject = {
package : {
data : $('#packagefileBrowser').data('base64'),
encoding : 'base64'
},
attachments : [],
schema : $('#messageImport_schema option:selected').val(),
filename : $('#packagefileBrowser').data('filename')
};
var attachments = $('#attachmentsContainer').find('input');
$.each(attachments, function() {
var attachment = {
filename : $(this).val(),
data : $(this).data('base64')
};
jsonObject.attachments.push(attachment);
})
return jsonObject;
}
function serializeParams(formData) {
var paramsInputs = $('#userInputs').find('input');
var paramsSelects = $('#userInputs').find('select');