Commit c845ee8d authored by Szabolcs Gyurko's avatar Szabolcs Gyurko
Browse files

Fixed issues around path delimiter. Added support for the content-disposition header.

parent 10dc6dd4
Pipeline #335 passed with stage
in 8 minutes and 44 seconds
......@@ -76,7 +76,7 @@ class AddJob @Inject() (val controllerComponents: ControllerComponents, protecte
if (jobData.pass.nonEmpty && jobData.pass.get == Defaults.PASSWORD_PLACEHOLDER)
jobData = addJobData.copy(pass = sget2.settings.getOrElse(Defaults.SETTING_PASSWORD, None))
val path = s"${jobData.path}/${new java.net.URL(jobData.url).getFile.replaceAll(".*/([^/]+)$", "$1")}"
val path = s"${jobData.path}${java.io.File.separator}${new java.net.URL(jobData.url).getFile.replaceAll(".*/([^/]+)$", "$1")}"
val jobRecord = JobRecord(0, jobData.url, path, jobData.maxBandwidth,
jobData.maxThreads, jobData.user, jobData.pass)
......
package sget2
import java.net.URL
import org.slf4j.LoggerFactory
import scala.io.Codec
......@@ -15,6 +17,10 @@ abstract class AbstractWorker(protected val downloadJob: DownloadJob, protected
private val threadGroup = new ThreadGroup(downloadJob.id.toString)
private val Logger = LoggerFactory.getLogger(getClass)
override def getArtifactName(url: URL): String = {
url.getFile.replaceAll(".*/([^/]+)$", "$1")
}
/**
* Generic worker logic implementation
*/
......@@ -26,6 +32,12 @@ abstract class AbstractWorker(protected val downloadJob: DownloadJob, protected
downloadJob.size = getArtifactSize(downloadJob.url)
Logger.debug(s"Artifact size looks like ${downloadJob.size} bytes")
val artifactName = getArtifactName(downloadJob.url)
if (!downloadJob.path.endsWith(artifactName)) {
Logger.debug(s"Artifact has a name reported: ${artifactName}")
downloadJob.path = downloadJob.path.replaceAll(s"^(.*)(${java.io.File.separator})[^${java.io.File.separator}]+$$", s"$$1$$2${artifactName}")
}
if (downloadJob.size > 0) {
downloadJob.initialize(downloadJob.size)
......
......@@ -67,7 +67,7 @@ class DownloadJob (private var _url: URL, private var _path: String)(implicit pr
/* Initialize target file, if not yet done */
if (!targetFileInitialized) {
if (size != 0 && (!fileTest.exists() || fileTest.length() != size)) {
Logger.debug(s"Initializing file ${path} with size {$size}")
Logger.debug(s"Initializing file ${path} with size ${size}")
val file = new RandomAccessFile(path, "rws")
file.seek(size - 1)
file.writeByte(0)
......@@ -244,4 +244,8 @@ class DownloadJob (private var _url: URL, private var _path: String)(implicit pr
_id = arg
sget2.postEvent(PersistEvent())
}
def path_=(arg: String) = {
_path = arg
sget2.postEvent(PersistEvent())
}
}
......@@ -12,6 +12,9 @@ trait Worker extends Runnable {
/** implements the artifact size retrieval. Needed in the worker to initialize job. */
def getArtifactSize(url: URL): Long
/** implements the artifact name retrieval. Some servers return the name in Content-Disposition header */
def getArtifactName(url: URL): String
/** returns a thread implementation which will download a slot of a job */
def workerThread(slot: (Long, Long)): Runnable
}
......@@ -25,6 +25,41 @@ class HttpWorker (override protected val downloadJob: DownloadJob,
override def isTrusted(chain: Array[X509Certificate], authType: String): Boolean = true
}).build())
override def getArtifactName(url: URL): String = {
val requestConfig = RequestConfig.custom().setSocketTimeout(15000).setConnectTimeout(5000).build()
var httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).disableContentCompression()
.setSSLSocketFactory(sslConnectionSocketFactory).build()
val port = { if (url.getPort == -1) url.getDefaultPort else url.getPort }
val method = new HttpHead(url.toURI)
try {
method.addHeader("Connection", "close")
method.addHeader("Range", "bytes=0-")
if (downloadJob.username.nonEmpty && downloadJob.password.nonEmpty) {
val credentialsProvider = new BasicCredentialsProvider()
credentialsProvider.setCredentials(
new AuthScope(url.getHost, port, url.getHost),
new UsernamePasswordCredentials(downloadJob.username.get, downloadJob.password.get))
httpClient = HttpClients.custom().setDefaultCredentialsProvider(credentialsProvider).setDefaultRequestConfig(requestConfig).build()
}
val httpResponse = httpClient.execute(method)
val contentDispositionHeader = httpResponse.getFirstHeader("Content-Disposition")
if (contentDispositionHeader == null) {
return super.getArtifactName(url)
}
contentDispositionHeader.getValue.replaceAll("attachment; .*filename=\"([^\"]+)\".*", "$1")
} catch {
case e: Exception
Logger.warn(s"Error communicating to ${url.getHost}:${port}, exception: ${e.getClass.getSimpleName}: ${e.getMessage}")
super.getArtifactName(url)
} finally {
method.reset()
httpClient.close()
}
}
/**
* HTTP specific file size retrieval using the HEAD method.
*
......
name := "sget2"
version := "1.2.2"
version := "1.3"
lazy val `sget2` = (project in file(".")).enablePlugins(PlayScala)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment