Java FTPS client through HTTP proxy

Question

I'm trying to develop a Java FTPS client using Apache Commons Net library, based on apache example and FTPSClient class. To run de code I'm using Java 8, update 45.

The exception occurs when I'm invoking the method "retrieveFile". I'm not sure, but I belive the connection used to tranfer the file is not using the HTTP proxy specified above.

With FileZilla client I can tranfer files using the same configurations.

How can I fix this problem?

My code:

// client with explicit security
FTPSClient ftps = new FTPSClient(false);
// HTTP proxy configuration
Proxy proxy = new Proxy(Type.HTTP, new InetSocketAddress("<REMOVED_FOR_SERCURITY>", <REMOVED_FOR_SERCURITY>));
ftps.setProxy(proxy);
// to show FTP commands in prompt
ftps.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out), true));
// disable remote host verification
ftps.setRemoteVerificationEnabled(false);
// trust in ALL
ftps.setTrustManager(TrustManagerUtils.getAcceptAllTrustManager());
// send keepAlive every 30 seconds
ftps.setControlKeepAliveTimeout(10l);
// data transfer timeout
ftps.setDataTimeout(30000);

// connect
ftps.connect("<REMOVED_FOR_SERCURITY>", <REMOVED_FOR_SERCURITY>);
ftps.login("<REMOVED_FOR_SERCURITY>", "<REMOVED_FOR_SERCURITY>");

// config
ftps.setCharset(Charset.forName("UTF-8"));
ftps.setBufferSize(0);
ftps.setFileType(FTP.BINARY_FILE_TYPE);
ftps.enterLocalPassiveMode();
ftps.execPROT("P");

// ... do some operations
ftps.retrieveFile("/dir1/dir2/fileX.zip", new ByteArrayOutputStream());

// close
ftps.logout();
ftps.disconnect();

The output:

220 (vsFTPd 2.2.2)
AUTH TLS
234 Proceed with negotiation.
USER *******
331 Please specify the password.
PASS *******
230 Login successful.
TYPE I
200 Switching to Binary mode.
PROT P
200 PROT now Private.
PASV
227 Entering Passive Mode (<REMOVED_FOR_SERCURITY>).
Exception in thread "main" java.net.ConnectException: Connection timed out: connect
    at java.net.DualStackPlainSocketImpl.connect0(Native Method)
    at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:345)
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    at java.net.Socket.connect(Socket.java:589)
    at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:656)
    at org.apache.commons.net.ftp.FTPClient._openDataConnection_(FTPClient.java:894)
    at org.apache.commons.net.ftp.FTPSClient._openDataConnection_(FTPSClient.java:600)
    at org.apache.commons.net.ftp.FTPClient._retrieveFile(FTPClient.java:1854)
    at org.apache.commons.net.ftp.FTPClient.retrieveFile(FTPClient.java:1845)
    at br.com.bat.crm.test.util.FTPSClientTest.main(FTPSClientTest.java:57)

Answer 1

I downloaded the source code of commons-net 3.3 and implemented my own FTPS through HTTP Proxy client. It has a problem when calling the method keepAlive, is giving the exception "java.net.SocketTimeoutException: Read timed out". I don't know what is causing this error. For me is not a problem, because I'm not using this feature.

Added in class FTPClient:

protected int getDataTimeout() {
    return __dataTimeout;
}

Added in class FTPSClient:

protected SSLContext getContext() {
    return context;
}

Created the class FTPSHTTPClient:

package org.apache.commons.net.ftp;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.Inet6Address;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.List;

import javax.net.ssl.SSLSocket;

import org.apache.commons.net.util.Base64;

/**
 * Experimental attempt at FTPS client that tunnels over an HTTP proxy connection.
 *
 * @author TECBMJNA
 * @created 22/07/2015 09:29:45
 */
public class FTPSHTTPClient extends FTPSClient {

    private final String proxyHost;
    private final int proxyPort;
    private final String proxyUsername;
    private final String proxyPassword;

    private String tunnelHost; // Save the host when setting up a tunnel (needed for EPSV)

    private static final byte[] CRLF = { '\r', '\n' };
    private final Base64 base64 = new Base64();

    /**
     * Constructor with proxy authentication
     *
     * @param proxyHost
     * @param proxyPort
     * @param proxyUser
     * @param proxyPass
     *
     * @author TECBMJNA
     * @created 22/07/2015 10:06:04
     */
    public FTPSHTTPClient(String proxyHost, int proxyPort, String proxyUser, String proxyPass) {
        super();
        this.proxyHost = proxyHost;
        this.proxyPort = proxyPort;
        this.proxyUsername = proxyUser;
        this.proxyPassword = proxyPass;
        this.tunnelHost = null;

        //TECBMJNA so funciona a partir do Java 8, pois Java 8 aceita proxy HTTP
        //setProxy(new Proxy(Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)));
    }

    /**
     * Default constructor
     *
     * @param proxyHost
     * @param proxyPort
     *
     * @author TECBMJNA
     * @created 22/07/2015 10:06:14
     */
    public FTPSHTTPClient(String proxyHost, int proxyPort) {
        this(proxyHost, proxyPort, null, null);
    }

    /**
     *
     * @see org.apache.commons.net.ftp.FTPSClient#_openDataConnection_(java.lang.String, java.lang.String)
     */
    @Override
    protected Socket _openDataConnection_(String command, String arg) throws IOException {
        //Force local passive mode, active mode not supported by through proxy
        if (getDataConnectionMode() != PASSIVE_LOCAL_DATA_CONNECTION_MODE) {
            throw new IllegalStateException("Only passive connection mode supported");
        }

        final boolean isInet6Address = getRemoteAddress() instanceof Inet6Address;
        String passiveHost = null;

        // Try EPSV command first on IPv6 - and IPv4 if enabled.
        // When using IPv4 with NAT it has the advantage
        // to work with more rare configurations.
        // E.g. if FTP server has a static PASV address (external network)
        // and the client is coming from another internal network.
        // In that case the data connection after PASV command would fail,
        // while EPSV would make the client succeed by taking just the port.
        boolean attemptEPSV = isUseEPSVwithIPv4() || isInet6Address;

        if (attemptEPSV && epsv() == FTPReply.ENTERING_EPSV_MODE) {
            _parseExtendedPassiveModeReply(_replyLines.get(0));
            passiveHost = this.tunnelHost;
        } else {
            if (isInet6Address) {
                return null; // Must use EPSV for IPV6
            }

            // If EPSV failed on IPV4, revert to PASV
            if (pasv() != FTPReply.ENTERING_PASSIVE_MODE) {
                return null;
            }

            _parsePassiveModeReply(_replyLines.get(0));
            passiveHost = this.getPassiveHost();
        }

        Socket proxySocket = new Socket();

        if (getReceiveDataSocketBufferSize() > 0) {
            proxySocket.setReceiveBufferSize(getReceiveDataSocketBufferSize());
        }

        if (getSendDataSocketBufferSize() > 0) {
            proxySocket.setSendBufferSize(getSendDataSocketBufferSize());
        }

        if (getPassiveLocalIPAddress() != null) {
            proxySocket.bind(new InetSocketAddress(getPassiveLocalIPAddress(), 0));
        }

        if (getDataTimeout() >= 0) {
            proxySocket.setSoTimeout(getDataTimeout());
        }

        proxySocket.connect(new InetSocketAddress(proxyHost, proxyPort), getConnectTimeout());

        tunnelHandshake(passiveHost, this.getPassivePort(), proxySocket.getInputStream(),
                        proxySocket.getOutputStream());

        Socket socket = getContext().getSocketFactory().createSocket(proxySocket, passiveHost,
                                                                     this.getPassivePort(), true);

        if (getReceiveDataSocketBufferSize() > 0) {
            socket.setReceiveBufferSize(getReceiveDataSocketBufferSize());
        }

        if (getSendDataSocketBufferSize() > 0) {
            socket.setSendBufferSize(getSendDataSocketBufferSize());
        }

        if (getPassiveLocalIPAddress() != null) {
            socket.bind(new InetSocketAddress(getPassiveLocalIPAddress(), 0));
        }

        if (getDataTimeout() >= 0) {
            socket.setSoTimeout(getDataTimeout());
        }

        if ((getRestartOffset() > 0) && !restart(getRestartOffset())) {
            proxySocket.close();
            socket.close();
            return null;
        }

        if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) {
            proxySocket.close();
            socket.close();
            return null;
        }

        if (socket instanceof SSLSocket) {
            SSLSocket sslSocket = (SSLSocket) socket;

            sslSocket.setUseClientMode(getUseClientMode());
            sslSocket.setEnableSessionCreation(getEnableSessionCreation());

            // server mode
            if (!getUseClientMode()) {
                sslSocket.setNeedClientAuth(getNeedClientAuth());
                sslSocket.setWantClientAuth(getWantClientAuth());
            }

            if (getEnabledCipherSuites() != null) {
                sslSocket.setEnabledCipherSuites(getEnabledCipherSuites());
            }

            if (getEnabledProtocols() != null) {
                sslSocket.setEnabledProtocols(getEnabledProtocols());
            }
            sslSocket.startHandshake();
        }

        return socket;
    }

    /**
     *
     * @see org.apache.commons.net.SocketClient#connect(java.lang.String, int)
     */
    @Override
    public void connect(String host, int port) throws SocketException, IOException {

        _socket_ = new Socket(proxyHost, proxyPort);
        _input_ = _socket_.getInputStream();
        _output_ = _socket_.getOutputStream();

        try {
            tunnelHandshake(host, port, _input_, _output_);
        } catch (Exception e) {
            IOException ioe = new IOException("Could not connect to " + host + " using port " + port);
            ioe.initCause(e);
            throw ioe;
        }

        super._connectAction_();
    }

    /**
     * Tunnels FTPS client connection over an HTTP proxy connection
     *
     * @param host
     * @param port
     * @param input
     * @param output
     * @throws IOException
     * @throws UnsupportedEncodingException
     *
     * @author TECBMJNA
     * @created 22/07/2015 09:32:23
     */
    private void tunnelHandshake(String host, int port, InputStream input, OutputStream output) throws IOException,
            UnsupportedEncodingException {
        final String connectString = "CONNECT " + host + ":" + port + " HTTP/1.1";
        final String hostString = "Host: " + host + ":" + port;

        this.tunnelHost = host;
        output.write(connectString.getBytes("UTF-8")); // TODO what is the correct encoding?
        output.write(CRLF);
        output.write(hostString.getBytes("UTF-8"));
        output.write(CRLF);

        if (proxyUsername != null && proxyPassword != null) {
            final String auth = proxyUsername + ":" + proxyPassword;
            final String header = "Proxy-Authorization: Basic " + base64.encodeToString(auth.getBytes("UTF-8"));
            output.write(header.getBytes("UTF-8"));
        }

        output.write(CRLF);
        output.flush();

        List<String> response = new ArrayList<String>();
        BufferedReader reader = new BufferedReader(new InputStreamReader(input, getCharsetName())); // Java 1.6 can use getCharset()

        for (String line = reader.readLine(); line != null && line.length() > 0; line = reader.readLine()) {
            response.add(line);
        }

        int size = response.size();

        if (size == 0) {
            throw new IOException("No response from proxy");
        }

        String code = null;
        String resp = response.get(0);

        if (resp.startsWith("HTTP/") && resp.length() >= 12) {
            code = resp.substring(9, 12);
        } else {
            throw new IOException("Invalid response from proxy: " + resp);
        }

        if (!"200".equals(code)) {
            StringBuilder msg = new StringBuilder();
            msg.append("HTTPTunnelConnector: connection failed\r\n");
            msg.append("Response received from the proxy:\r\n");

            for (String line : response) {
                msg.append(line);
                msg.append("\r\n");
            }

            throw new IOException(msg.toString());
        }
    }

}

The above code can be tested with this code:

    // client with explicit security, TLS protocol and tunneled over HTTP proxy
    FTPSHTTPClient ftps = new FTPSHTTPClient(<REMOVED_FOR_SERCURITY>, <REMOVED_FOR_SERCURITY>);
    // to show FTP commands in prompt
    ftps.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out), true));
    // disable remote host verification
    ftps.setRemoteVerificationEnabled(false);
    // trust in ALL
    ftps.setTrustManager(TrustManagerUtils.getAcceptAllTrustManager());
    // data transfer timeout
    ftps.setDataTimeout(1800000); // 30 

    // keepAlive - DON'T USE, HAS A BUG WITH HTTP PROXY - java.net.SocketTimeoutException: Read timed out
    //ftps.setControlKeepAliveTimeout(10l);

    // connect
    ftps.connect(<REMOVED_FOR_SERCURITY>, <REMOVED_FOR_SERCURITY>);
    ftps.login(<REMOVED_FOR_SERCURITY>, <REMOVED_FOR_SERCURITY>);

    // config
    ftps.setCharset(Charset.forName("UTF-8"));
    ftps.setBufferSize(0);
    ftps.execPROT("P");
    ftps.setFileType(FTP.BINARY_FILE_TYPE);
    ftps.enterLocalPassiveMode();

    // ... do some operations
    ftps.changeWorkingDirectory(<REMOVED_FOR_SERCURITY>);
    ftps.storeFile(<REMOVED_FOR_SERCURITY>, new FileInputStream(<REMOVED_FOR_SERCURITY>));

    BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(<REMOVED_FOR_SERCURITY>));
    ftps.retrieveFile(<REMOVED_FOR_SERCURITY>, outputStream);
    outputStream.close();

    // close
    ftps.logout();
    ftps.disconnect();