Unable to get response from an url which requires user acceptance before it redirects to target url

Question

I had a job which has to download a file from "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt" and place in the folder. Currently we are doing it manually as job runs for once in 15 days but i want to download the file using the program before job process the file.

The problem here is, when we hit the url (https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt) on the browser, on first time it will be redirected to "https://www.frbservices.org/EPaymentsDirectory/agreement.html", on clicking "Agree" button then it will redirects to "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt". On clicking Agree button JSESSIONID is saved as a cookie in the browser. when we click the target url next time it directly opens the required page with out any agree of terms..

I tried below thing to get the response from target url but not able to achieve the response as expected..

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;


public class HttpTest {

    static String cookies;
    public static void main(String [] args) throws ClientProtocolException, IOException {
        String url = "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt";
        String agreementURL = "https://www.frbservices.org/EPaymentsDirectory/submitAgreement";
        String USER_AGENT = "Mozilla/5.0";
        CookieHandler.setDefault(new CookieManager());

        HttpClient client = HttpClientBuilder.create().build();
        String result;
        result =doPost(agreementURL, USER_AGENT, client);
        System.out.println(result);

        result = doGet(url, USER_AGENT, client);
        System.out.println("result:"+result);
        //String result =doGet(url, USER_AGENT, client);
        if (result != null) {

            Document doc = Jsoup.parse(result.toString());

            // 
            Element agreeElement = doc.getElementById("agree_terms_use");


        }

    }
    public static String getCookies() {
        return cookies;
    }



  public static  void setCookies(String cookies) {
        HttpTest.cookies = cookies;
    }

    public static String doGet(String url , String USER_AGENT, HttpClient client) throws ClientProtocolException, IOException {
        HttpGet request = new HttpGet(url);

        request.setHeader("User-Agent", USER_AGENT);
        request.setHeader("Accept",
            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        request.setHeader("Accept-Language", "en-US,en;q=0.5");
        System.out.println("c"+getCookies());
        request.setHeader("Cookie", getCookies());

        HttpResponse response = client.execute(request);
        int responseCode = response.getStatusLine().getStatusCode();

        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : " + responseCode);
        System.out.println(response);
        BufferedReader rd = new BufferedReader(
                new InputStreamReader(response.getEntity().getContent()));

        StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null) {
            result.append(line);
        }
        return result.toString();
    }

    public static String doPost(String url , String USER_AGENT, HttpClient client) throws ClientProtocolException, IOException {
        HttpPost post = new HttpPost(url);

        post.setHeader("Host", "www.frbservices.org");
        post.setHeader("User-Agent", USER_AGENT);
        post.setHeader("Accept",
                 "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        post.setHeader("Accept-Encoding","gzip, deflate, br");
        post.setHeader("Accept-Language", "en-US,en;q=0.5");
        post.setHeader("Location", "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt");
        post.setHeader("Connection", "keep-alive");
        post.setHeader("Referer", "https://www.frbservices.org/EPaymentsDirectory/agreement.html");
        post.setHeader("Content-Type", "application/x-www-form-urlencoded");
        post.setHeader("origin", "https://www.frbservices.org");
        post.setHeader("Upgrade-Insecure-Requests","1");
        //post.setHeader("agreementValue","Agree");

        List<BasicNameValuePair> paramList = new ArrayList<BasicNameValuePair>();
        paramList.add(new BasicNameValuePair("agreementValue", "Agree"));

        post.setEntity(new UrlEncodedFormEntity(paramList));


        HttpResponse response = client.execute(post);

        int responseCode = response.getStatusLine().getStatusCode();

        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : " + responseCode);
        System.out.println("Response: " + response);
        BufferedReader rd = new BufferedReader(
                new InputStreamReader(response.getEntity().getContent()));

        setCookies(response.getFirstHeader("Set-Cookie") == null ? "" :
            response.getFirstHeader("Set-Cookie").toString());

        StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null) {
            result.append(line);
        }
        return result.toString();
    }

}