From f223dd5110525470975e6a0db16f22eda11554a9 Mon Sep 17 00:00:00 2001 From: mykola2312 <49044616+mykola2312@users.noreply.github.com> Date: Fri, 19 Apr 2024 19:02:27 +0300 Subject: [PATCH] implement some verbose classes for dealing with web requests --- src/main/java/com/mykola2312/mptv/Main.java | 2 +- .../mptv/crawler/HttpClientFactory.java | 36 +++++++++------ .../mykola2312/mptv/crawler/WebContent.java | 26 +++++++++++ .../mykola2312/mptv/crawler/WebException.java | 44 +++++++++++++++++++ .../mykola2312/mptv/crawler/WebRequest.java | 17 +++---- 5 files changed, 102 insertions(+), 23 deletions(-) create mode 100644 src/main/java/com/mykola2312/mptv/crawler/WebContent.java create mode 100644 src/main/java/com/mykola2312/mptv/crawler/WebException.java diff --git a/src/main/java/com/mykola2312/mptv/Main.java b/src/main/java/com/mykola2312/mptv/Main.java index 4a49533..d978858 100644 --- a/src/main/java/com/mykola2312/mptv/Main.java +++ b/src/main/java/com/mykola2312/mptv/Main.java @@ -91,7 +91,7 @@ public class Main { System.out.printf("fetchOne -> %d: %s\n", test.id, test.value); WebRequest get = new WebRequest("https://example.com"); - System.out.println(get.fetch()); + System.out.println(get.fetch().body); logger.info("mptv started"); } diff --git a/src/main/java/com/mykola2312/mptv/crawler/HttpClientFactory.java b/src/main/java/com/mykola2312/mptv/crawler/HttpClientFactory.java index 5e3c8c7..cbfbe53 100644 --- a/src/main/java/com/mykola2312/mptv/crawler/HttpClientFactory.java +++ b/src/main/java/com/mykola2312/mptv/crawler/HttpClientFactory.java @@ -1,5 +1,7 @@ package com.mykola2312.mptv.crawler; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import javax.net.ssl.SSLContext; @@ -23,25 +25,31 @@ public class HttpClientFactory { "DTLSv1.0", }; - public static HttpClient getHttpsClient() throws Exception { + public static HttpClient getHttpsClient() throws WebException { if (client != null) { return client; } - SSLContext sslContext = SSLContexts - .custom() - .build(); - sslContext.init(null, new TrustManager[] { new HttpsTrustManager() }, new SecureRandom()); + try { + SSLContext sslContext = SSLContexts + .custom() + .build(); + sslContext.init(null, new TrustManager[] { new HttpsTrustManager() }, new SecureRandom()); - SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, - SUPPORTED_PROTOCOLS, - null, - SSLConnectionSocketFactory.getDefaultHostnameVerifier() - ); - client = HttpClients - .custom() - .setSSLSocketFactory(factory) - .build(); + SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, + SUPPORTED_PROTOCOLS, + null, + SSLConnectionSocketFactory.getDefaultHostnameVerifier() + ); + client = HttpClients + .custom() + .setSSLSocketFactory(factory) + .build(); + } catch (NoSuchAlgorithmException e) { + throw new WebException(e); + } catch (KeyManagementException e) { + throw new WebException(e); + } return client; } diff --git a/src/main/java/com/mykola2312/mptv/crawler/WebContent.java b/src/main/java/com/mykola2312/mptv/crawler/WebContent.java new file mode 100644 index 0000000..c2d8f05 --- /dev/null +++ b/src/main/java/com/mykola2312/mptv/crawler/WebContent.java @@ -0,0 +1,26 @@ +package com.mykola2312.mptv.crawler; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.util.EntityUtils; +import org.apache.log4j.Logger; + +public class WebContent { + public int status; + public String body = null; + + private static final Logger logger = Logger.getLogger(WebContent.class); + + public WebContent(HttpResponse response) { + this.status = response.getStatusLine().getStatusCode(); + HttpEntity entity = response.getEntity(); + if (entity != null) { + try { + this.body = EntityUtils.toString(entity, "UTF-8"); + } catch (Exception e) { + logger.warn(String.format("failed to get content for %s: %s", response.toString())); + logger.warn(e); + } + } + } +} diff --git a/src/main/java/com/mykola2312/mptv/crawler/WebException.java b/src/main/java/com/mykola2312/mptv/crawler/WebException.java new file mode 100644 index 0000000..63201f4 --- /dev/null +++ b/src/main/java/com/mykola2312/mptv/crawler/WebException.java @@ -0,0 +1,44 @@ +package com.mykola2312.mptv.crawler; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; + +import org.apache.http.client.ClientProtocolException; +import org.apache.log4j.Logger; + +// TODO: make more useful this stub exception +public class WebException extends RuntimeException { + public enum Type { + FAILED_TO_MAKE_CLIENT, + FETCH_FAILURE, + IO_ERROR, + }; + + public final Type type; + + protected static final Logger logger = Logger.getLogger(WebException.class); + protected void log() { + logger.warn(String.format("%s: %s", type.name(), this.getCause().getMessage())); + } + + public WebException(NoSuchAlgorithmException e) { + super(Type.FAILED_TO_MAKE_CLIENT.name(), e); + this.type = Type.FAILED_TO_MAKE_CLIENT; + } + + public WebException(KeyManagementException e) { + super(Type.FAILED_TO_MAKE_CLIENT.name(), e); + this.type = Type.FAILED_TO_MAKE_CLIENT; + } + + public WebException(ClientProtocolException e) { + super(Type.FETCH_FAILURE.name(), e); + this.type = Type.FETCH_FAILURE; + } + + public WebException(IOException e) { + super(Type.IO_ERROR.name(), e); + this.type = Type.IO_ERROR; + } +} diff --git a/src/main/java/com/mykola2312/mptv/crawler/WebRequest.java b/src/main/java/com/mykola2312/mptv/crawler/WebRequest.java index 2173f6a..64fbd52 100644 --- a/src/main/java/com/mykola2312/mptv/crawler/WebRequest.java +++ b/src/main/java/com/mykola2312/mptv/crawler/WebRequest.java @@ -1,10 +1,11 @@ package com.mykola2312.mptv.crawler; -import org.apache.http.HttpEntity; +import java.io.IOException; + import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; -import org.apache.http.util.EntityUtils; public class WebRequest { private final HttpGet httpGet; @@ -13,16 +14,16 @@ public class WebRequest { httpGet = new HttpGet(url); } - public String fetch() { + public WebContent fetch() throws WebException { try { HttpClient client = HttpClientFactory.getHttpsClient(); HttpResponse response = client.execute(httpGet); - HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, "UTF-8"); - } catch (Exception e) { - e.printStackTrace(); - return null; + return new WebContent(response); + } catch (ClientProtocolException e) { + throw new WebException(e); + } catch (IOException e) { + throw new WebException(e); } } }