package com.example.service import android.net.Uri import android.util.Log import com.example.action.BaseAction import com.example.action.HttpActionRequest import com.example.action.HttpMethod import com.example.action.NameValue import com.example.action.NameVariable import com.example.action.Next import com.example.http.HttpClient import com.example.http.HttpClient.call import com.example.http.Request import com.example.http.Response import com.example.logger.LogUtils import com.example.report.ActionExec import com.example.task.TaskConfig import com.example.utils.toJsonString import com.example.utils.toJsonString1 import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.net.URI import java.net.URL import kotlin.time.DurationUnit import kotlin.time.toDuration class HttpService( private val action: BaseAction.HttpAction, override val taskConfig: TaskConfig ) : BaseService(taskConfig) { companion object { private const val MAX_REDIRECT_COUNT = 30 val pattern = Regex("^[a-zA-Z0-9]+://.*") } override suspend fun execute(onFinish: (List) -> Unit) = withContext(Dispatchers.Default) { val actionExecList = mutableListOf() var currentStep = taskConfig.currentStep try { LogUtils.info("action delay: ${action.delay} s, it's async: ${action.async}") if (action.delay > 0) { delay(action.delay.toDuration(DurationUnit.SECONDS)) } val actionRequest = action.request ?: throw NullPointerException("request is null") amendActionRequest(actionRequest) var httpRequest = actionRequest.buildHttpRequest() var httpResponse: Response? = null var proceedTask = false var stepCallHttpCount = 1 if (action.async) { httpRequest.makeAsyncRequest() val actionExec = httpRequest.genActionExec(null, stepCallHttpCount) actionExec.respCode = ASYNC_EXEC_CODE actionExecList += actionExec proceedTask = true } else { httpResponse = httpRequest.call() val actionExec = httpRequest.genActionExec(httpResponse, stepCallHttpCount) actionExecList += actionExec when (httpResponse.code) { in 100 until 300 -> { proceedTask = true } in 300 until 400 -> { var redirectUrl: String? = httpResponse.headers.get( HttpClient.Params.REQUEST_HEADER_LOCATION, ignoreCase = true )?.firstOrNull() while ((isActive && httpResponse != null && httpResponse.code in (300 until 400) && stepCallHttpCount <= MAX_REDIRECT_COUNT && !redirectUrl.isNullOrBlank()) && redirectUrl.isHttpRedirect() ) { runCatching { stepCallHttpCount++ httpRequest = httpRequest.buildRedirectHttpRequest( redirectUrl!!, httpRequest.url ) httpResponse = httpRequest.call() if (httpResponse!!.code !in 300 until 400) { if (!httpResponse?.headers.isNullOrEmpty()) { httpResponse?.headers = httpResponse!!.headers.toMutableMap().apply { put( HttpClient.Params.REQUEST_HEADER_REFERER, listOf( httpRequest.headers.getOrDefault( HttpClient.Params.REQUEST_HEADER_REFERER, ignoreCase = true, "" ) ) ) } } } redirectUrl = httpResponse?.headers?.get( HttpClient.Params.REQUEST_HEADER_LOCATION, ignoreCase = true )?.firstOrNull() LogUtils.info("redirectUrl: $redirectUrl") proceedTask = true httpRequest.genActionExec(httpResponse, stepCallHttpCount) }.onFailure { e -> LogUtils.error(throwable = e) proceedTask = action.skipError actionExecList += httpRequest.genActionExec( null, stepCallHttpCount ).apply { respCode = HttpClient.ErrorCode.ERROR_CODE_HTTP_BUILD_CONNECTION_FAILED } }.onSuccess { actionExecList += it } } } else -> { proceedTask = action.skipError } } } if (proceedTask) { httpResponse?.apply { extractResponseVariableToCache(action, httpRequest, httpResponse) val nextStep = action.next.httpGetNextStepIndex( httpRequest, httpResponse, currentStep ) taskConfig.currentStep = nextStep } ?: let { taskConfig.currentStep = ++currentStep } } else { taskConfig.currentStep = Int.MAX_VALUE } } catch (e: Exception) { LogUtils.error(throwable = e) val actionExec = genExceptionActionExec( action, ERROR_CODE_HTTP_ACTION_EXEC_FAILED, Log.getStackTraceString(e) ) actionExecList += actionExec if (action.skipError) { taskConfig.currentStep = ++currentStep } else { taskConfig.currentStep = Int.MAX_VALUE } } LogUtils.info("finish action: ${action.request?.url}") onFinish(actionExecList) } private fun Request.genActionExec( httpResponse: Response?, redirectCount: Int ): ActionExec { val actionExec = ActionExec( step = taskConfig.currentStep, index = redirectCount, time = System.currentTimeMillis(), url = url, method = method?.value ?: "GET", reqHeader = headers.toJsonString1() ) if (body.isNotEmpty()) { actionExec.reqData = String(body) } httpResponse?.let { response -> if (response.headers.isNotEmpty()) { kotlin.runCatching { URL(url).apply { URI(protocol, host, path, query, null).let { uri -> taskConfig.cookieManager.put(uri, response.headers) } } }.onFailure { LogUtils.error(throwable = it) } actionExec.respHeader = response.headers.toJsonString() } actionExec.respCode = response.code if (response.data.isNotEmpty()) { actionExec.respData = String(response.data) } actionExec.cost = httpResponse.endTime - httpResponse.startTime } return actionExec } private fun Request.makeAsyncRequest() = scope.launch { var stepCallHttpCount = 0 var asyncRequest: Request = copy() var asyncResponse = asyncRequest.call() var locationList: MutableList var redirectUrl: String? = null if (asyncResponse.code in 300 until 400) { locationList = asyncResponse.headers.get( HttpClient.Params.REQUEST_HEADER_LOCATION, ignoreCase = true )?.toMutableList() ?: mutableListOf() redirectUrl = locationList.firstOrNull() } while (asyncResponse.code in 300 until 400 && stepCallHttpCount <= MAX_REDIRECT_COUNT && !redirectUrl.isNullOrBlank() && redirectUrl.isHttpRedirect()) { kotlin.runCatching { stepCallHttpCount++ asyncRequest = asyncRequest.buildRedirectHttpRequest(redirectUrl!!, asyncRequest.url) asyncResponse = asyncRequest.call() locationList = asyncResponse.headers.get( HttpClient.Params.REQUEST_HEADER_LOCATION, ignoreCase = true )?.toMutableList() ?: mutableListOf() redirectUrl = locationList.firstOrNull() }.onFailure { LogUtils.error(throwable = it) } } } private fun String.isHttpRedirect(): Boolean = pattern.find(this)?.let { return@isHttpRedirect this.startsWith("http") } ?: true private fun HttpActionRequest.buildHttpRequest(): Request = Request( url = url, method = method, body = data.toByteArray(), headers = headers.nameValueToMap() ) private fun genWholeResponse(httpRequest: Request, httpResponse: Response?): String { return """ [${httpRequest.url}]${if (httpResponse?.data?.isNotEmpty() == true) String(httpResponse.data) else ""}[${ httpResponse?.headers?.let { if (it.isEmpty()) "" else it.toJsonString() } }] """.trimIndent() } private fun List.httpGetNextStepIndex( httpRequest: Request, httpResponse: Response?, currentStep: Int ): Int { val wholeResponse = genWholeResponse(httpRequest, httpResponse) return getNextStepIndex(wholeResponse, currentStep) } private fun extractResponseVariableToCache( action: BaseAction.HttpAction, httpRequest: Request, httpResponse: Response? ) { action.response?.let { actionResponse -> httpResponse?.headers?.let { responseHeaders -> extractCookieVariableToCache(actionResponse.cookies, responseHeaders) extractHeaderVariableToCache(actionResponse.headers, responseHeaders) extractBodyVariableToCache( action, genWholeResponse(httpRequest, httpResponse), httpResponse.data ) } } } private fun extractCookieVariableToCache( cookies: List, responseHeaders: Map> ) { if (cookies.isEmpty()) return runCatching { val cookieList = responseHeaders.get(HttpClient.Params.RESPONSE_HEADER_SET_COOKIE, ignoreCase = true) if (cookieList.isNullOrEmpty()) return cookies.map { nameVariable -> cookieList.map { cookie -> val cookieValues = cookie.split(";") cookieValues.map { cookieValue -> val keyPair = cookieValue.split("=", limit = 2) val key = keyPair.first().trim() val value = keyPair.getOrElse(1) { "" }.trim() if (key == nameVariable.name) { taskConfig.variableCache[nameVariable.variable] = value } } } } }.onFailure { LogUtils.error(throwable = it) } } private fun extractHeaderVariableToCache( headers: List, responseHeaders: Map> ) { if (headers.isEmpty() || responseHeaders.isEmpty()) return headers.map { nameVariable -> responseHeaders[nameVariable.name]?.firstOrNull()?.apply { taskConfig.variableCache[nameVariable.variable] = this } } } private fun amendActionRequest(actionRequest: HttpActionRequest) { actionRequest.headers.replaceVariableData() actionRequest.cookies.replaceVariableData() actionRequest.params.replaceVariableData() actionRequest.headers.amendBaseHeader(true).apply { actionRequest.headers = this.toMutableList() } if (actionRequest.data.isNotBlank()) { actionRequest.data = actionRequest.data.toVariableData() } actionRequest.url = when (actionRequest.method) { HttpMethod.Get -> { buildGetUrl(actionRequest) } else -> { actionRequest.url.toVariableData() } } if (actionRequest.url.startsWith("https", ignoreCase = true)) { actionRequest.headers.addSecChUa().apply { actionRequest.headers = this } } actionRequest.amendCookie() if (HttpMethod.Get != actionRequest.method) { val sendData: ByteArray = actionRequest.genPostData() if (sendData.isNotEmpty()) actionRequest.data = String(sendData) } } private fun buildGetUrl(actionRequest: HttpActionRequest): String { return actionRequest.url.toVariableData().let { u -> Uri.parse(u).buildUpon().apply { actionRequest.params.forEach { nameValue -> this.appendQueryParameter(nameValue.name, nameValue.value) } } }.build().toString() } private fun HttpActionRequest.amendCookie() { val cookies: MutableSet = mutableSetOf() cookieFromCookieManager(url).apply { if (autoCookie && isNotEmpty()) { cookies += this } } this.cookies.apply { if (isNotEmpty()) { forEach { cookies.remove(it) cookies += it } } } if (cookies.isNotEmpty()) { headers += cookies.toList().buildCookie() } } private fun HttpActionRequest.genPostData(): ByteArray { var sendData: ByteArray = byteArrayOf() if (params.isNotEmpty()) { sendData = params.joinToString("&") { "${it.name.urlEncode()}=${it.value.urlEncode()}" }.toByteArray() } if (data.isNotBlank()) { sendData = data.toByteArray(Charsets.UTF_8) } return sendData } private fun Request.buildRedirectHttpRequest(redirectUrl: String, originUrl: String): Request { val headers = this.genBaseHeaderMap().toMutableMap() headers[HttpClient.Params.REQUEST_HEADER_REFERER] = originUrl val request = Request( url = URL(URL(originUrl), redirectUrl.replace(" ", "%20")).toString(), method = HttpMethod.Get, headers = headers ) val cookies = cookieFromCookieManager(request.url) if (cookies.isNotEmpty()) { cookies.toList().buildCookie().let { cookie -> headers.put(cookie.name, cookie.value) } } if (request.url.isHttps()) { val secChUa = mutableListOf().addSecChUa() secChUa.forEach { headers[it.name] = it.value } } return request } } fun Map.get(key: String, ignoreCase: Boolean): V? { return this.entries.firstOrNull { it.key.equals(key, ignoreCase = ignoreCase) }?.value } fun Map.getOrDefault(key: String, ignoreCase: Boolean, defaultValue: V): V = this.get(key, ignoreCase) ?: defaultValue