使用htmlunit获取js渲染后的数据

pom.xml

@@ -159,9 +159,18 @@
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib</artifactId>
             <version>1.3.72</version>
+            <scope>compile</scope>
         </dependency>
 
 
+        <dependency>
+            <groupId>net.sourceforge.htmlunit</groupId>
+            <artifactId>htmlunit</artifactId>
+            <version>2.42.0</version>
+        </dependency>
+
+
+
     </dependencies>
 
     <build>
@@ -190,6 +199,20 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-maven-plugin</artifactId>
+                <version>1.3.72</version>
+                <executions>
+                    <execution>
+                        <id>compile</id>
+                        <phase>process-sources</phase>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

src/main/java/cn/celess/blog/util/HttpUtil.java

@@ -1,10 +1,15 @@
 package cn.celess.blog.util;
 
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.util.Objects;
 
 /**
@@ -62,4 +67,20 @@ public class HttpUtil {
         }
     }
 
+
+    public static String getAfterRendering(String url) {
+        try (final WebClient webClient = new WebClient(BrowserVersion.CHROME)) {
+            webClient.getOptions().setCssEnabled(false);
+            webClient.getOptions().setJavaScriptEnabled(true);
+            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
+            webClient.getOptions().setThrowExceptionOnScriptError(false);
+            webClient.getOptions().setDownloadImages(false);
+            webClient.getOptions().setActiveXNative(false);
+            webClient.setAjaxController(new NicelyResynchronizingAjaxController());
+            final HtmlPage page = webClient.getPage(url);
+            return page.asXml();
+        } catch (IOException e) {
+            return null;
+        }
+    }
 }