船舶大數(shù)據(jù)爬蟲采集長(zhǎng)效固定ip在java使用中的解決方案

時(shí)間:2022-08-15   來(lái)源:楓之葉網(wǎng)絡(luò)

隨著跨境電商的迅速發(fā)展，海運(yùn)船舶實(shí)時(shí)運(yùn)輸信息非常重要，很多船運(yùn)需要采用爬蟲技術(shù)進(jìn)行數(shù)據(jù)抓取并使用（注意此處的信息為公開非敏感信息哦）。分布式采集和監(jiān)控技術(shù)突飛猛進(jìn)。java語(yǔ)言經(jīng)常使用固定長(zhǎng)效ip進(jìn)行分布式數(shù)據(jù)采集和監(jiān)控。以下是demon：

    import java.io.ByteArrayOutputStream;

    import java.io.InputStream;

    import java.net.Authenticator;

    import java.net.HttpURLConnection;

    import java.net.InetSocketAddress;

    import java.net.PasswordAuthentication;

    import java.net.Proxy;

    import java.net.URL;

    class ProxyAuthenticator extends Authenticator {

        private String user, password;

        public ProxyAuthenticator(String user, String password) {

            this.user     = user;

            this.password = password;

        }

        protected PasswordAuthentication getPasswordAuthentication() {

            return new PasswordAuthentication(user, password.toCharArray());

        }

    }

    public class ProxyTest {

        public static void main(String args[]) throws Exception {

            String targetUrl = "http://www.xbltjxp.cn";

            System.setProperty("jdk.http.auth.tunneling.disabledSchemes", "");

            String proxyServer = "http://demon.pachongdaili.com";

            int proxyPort      = 65108;

            String proxyUser  = "user";

            String proxyPass  = "pwd";

            try {

                URL url = new URL(targetUrl);

                Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));

                InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);

                Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);

                HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);

                connection.setRequestProperty("Proxy-Switch-Ip","yes");

                byte[] response = readStream(connection.getInputStream());

                System.out.println(new String(response));

            } catch (Exception e) {

                System.out.println(e.getLocalizedMessage());

            }

        }

        public static byte[] readStream(InputStream inStream) throws Exception {

            ByteArrayOutputStream outSteam = new ByteArrayOutputStream();

            byte[] buffer = new byte[1024];

            int len = -1;

            while ((len = inStream.read(buffer)) != -1) {

                outSteam.write(buffer, 0, len);

            }

            outSteam.close();

            inStream.close();

            return outSteam.toByteArray();

        }

    }

              

當(dāng)前地址：http://www.xbltjxp.cn/support/a452.html 客服聯(lián)系QQ：475685360