gget/gget.c
2016-08-14 11:33:29 +02:00

773 lines
20 KiB
C

/*
Copyright 2014-2016 Grégory Soutadé
This file is part of gget.
gget is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
gget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with gget. If not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
#include <libgen.h>
#define DEFAULT_USER_AGENT "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0"
#define DEFAULT_NB_THREADS 3
#define MAX_NB_THREADS 10
#define DEFAULT_OUT_FILENAME "gget.out"
#ifdef WIN32
#include <windows.h>
#define GNU_ERR ""
typedef HANDLE pthread_t;
typedef void pthread_attr_t;
static int get_console_width()
{
RECT r;
HWND console = GetConsoleWindow();
GetWindowRect(console, &r);
return r.right-r.left;
}
static int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg)
{
*thread = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)start_routine,
arg, 0, NULL);
return 0;
}
static int pthread_join(pthread_t thread, void **retval)
{
WaitForSingleObject(thread, 0);
GetExitCodeThread(thread,(LPDWORD)retval);
CloseHandle(thread);
return 0;
}
#else
#include <sys/ioctl.h>
#include <pthread.h>
#define GNU_ERR " (%m)"
static int get_console_width()
{
struct winsize w;
ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
return w.ws_col;
}
#endif
typedef struct {
curl_off_t dltotal;
curl_off_t dlnow;
curl_off_t dllast;
unsigned speed;
} stats_t ;
typedef struct {
pthread_t thread;
char* tmp_filename;
//
CURL *curl;
char* url;
char* user_agent;
int fd;
int id;
unsigned already_downloaded;
unsigned start;
unsigned end;
unsigned max_chunk_size;
curl_off_t max_speed;
stats_t* stats;
} transfert_t;
typedef struct {
unsigned nb_threads;
stats_t* stats;
unsigned end;
} stats_params_t;
static void* display_stats(stats_params_t* p)
{
unsigned speed, percent, time_left, max_time_left;
unsigned total_percent, hours, minutes, seconds;
char* suffix;
stats_t* cur;
int i, nb_chars_printed;
while (!p->end)
{
// If the window has been resized
max_time_left = 0;
suffix = "B";
total_percent = 0;
nb_chars_printed = 0;
printf("\r");
for(i=0; i<p->nb_threads; i++)
{
cur = &p->stats[i];
speed = cur->speed;
if (cur->dltotal)
percent = (cur->dlnow*100)/cur->dltotal;
else
percent = 0;
if (speed)
time_left = (cur->dltotal-cur->dlnow)/speed;
else
time_left = 0;
if (speed < (1024*1024))
{
speed /= 1024;
suffix = "kB";
}
else
{
speed /= 1024*1024;
suffix = "MB";
}
nb_chars_printed +=
printf("T%d: %02u%% %u%s/s ",
i, percent, speed, suffix);
if (time_left > max_time_left)
max_time_left = time_left;
total_percent += percent/p->nb_threads;
}
nb_chars_printed += printf(" Total: %u%% ", total_percent);
if (max_time_left < 60)
{
nb_chars_printed += printf("eta %us", max_time_left);
}
else
{
seconds = max_time_left % 60;
max_time_left /= 60;
if (max_time_left < 60)
{
minutes = max_time_left;
nb_chars_printed += printf("eta %um %us", minutes, seconds);
}
else
{
minutes = max_time_left % 60;
hours = max_time_left / 60;
nb_chars_printed += printf("eta %uh %um %us", hours, minutes, seconds);
}
}
for(i=nb_chars_printed; i<get_console_width(); i++)
printf(" ");
fflush(stdout);
sleep(1);
}
return NULL;
}
static size_t write_cb(void *contents, size_t size, size_t nmemb, void *userp)
{
int ret;
ret = write(((transfert_t*)userp)->fd, contents, size*nmemb);
if (ret < 0)
printf("Error write" GNU_ERR "\n");
return ret;
}
static int progress_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
{
transfert_t* t = (transfert_t*)clientp;
double speed_d;
curl_easy_getinfo(t->curl, CURLINFO_SPEED_DOWNLOAD, &speed_d);
t->stats[t->id].dlnow += (dlnow - t->stats[t->id].dllast);
t->stats[t->id].dllast = dlnow;
t->stats[t->id].speed = speed_d;
return 0;
}
void* do_transfert(transfert_t* t)
{
CURLcode res;
char range[64];
unsigned start, end, chunk_size;
curl_easy_setopt(t->curl, CURLOPT_USERAGENT, t->user_agent);
curl_easy_setopt(t->curl, CURLOPT_URL, t->url);
curl_easy_setopt(t->curl, CURLOPT_FOLLOWLOCATION, 1L);
if (t->max_speed)
curl_easy_setopt(t->curl, CURLOPT_MAX_RECV_SPEED_LARGE, t->max_speed);
/* send all data to this function */
curl_easy_setopt(t->curl, CURLOPT_WRITEFUNCTION, write_cb);
/* we pass our 'chunk' struct to the callback function */
curl_easy_setopt(t->curl, CURLOPT_WRITEDATA, (void*)t);
curl_easy_setopt(t->curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(t->curl, CURLOPT_XFERINFOFUNCTION, progress_cb);
curl_easy_setopt(t->curl, CURLOPT_XFERINFODATA, t);
start = t->start;
if (t->max_chunk_size && (t->end - t->start) > t->max_chunk_size)
chunk_size = t->max_chunk_size;
else
chunk_size = t->end - t->start;
end = start + chunk_size;
while (start < t->end)
{
snprintf(range, sizeof(range), "%u-%u", start, end);
curl_easy_setopt(t->curl, CURLOPT_RANGE, range);
/* Perform the request, res will get the return code */
res = curl_easy_perform(t->curl);
/* Check for errors */
if(res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
break;
}
start += chunk_size + 1;
if (t->max_chunk_size && (t->end - start) > t->max_chunk_size)
chunk_size = t->max_chunk_size;
else
chunk_size = t->end - start;
end = start + chunk_size;
t->stats[t->id].dllast = 0;
}
return NULL;
}
static int configure_transfert(int id, transfert_t* t,
char* url, char* filename,
unsigned start, unsigned end,
unsigned max_speed, char* user_agent,
unsigned max_chunk_size, int* exists)
{
// filename + . + number + \0
unsigned filename_size = strlen(filename)+1+3+1;
struct stat s;
t->curl = curl_easy_init();
if(!t->curl)
return -1;
t->url = url;
t->tmp_filename = malloc(filename_size);
snprintf(t->tmp_filename, filename_size, "%s.%d",
filename, id);
t->id = id;
t->start = start;
t->end = end;
t->max_speed = max_speed;
t->user_agent = user_agent;
t->max_chunk_size = max_chunk_size;
if (stat(t->tmp_filename, &s))
{
*exists = 0;
t->fd = open(t->tmp_filename,
O_WRONLY | O_CREAT,
S_IRUSR|S_IWUSR);
t->already_downloaded = 0;
}
else
{
*exists = 1;
t->fd = open(t->tmp_filename,
O_WRONLY | O_APPEND);
t->start += s.st_size;
t->already_downloaded = s.st_size;
}
if (!t->fd)
{
printf("Opening '%s' failed" GNU_ERR "\n",
t->tmp_filename);
return -1;
}
return 0;
}
static int free_transfert(transfert_t* t)
{
if (t)
{
if (t->curl)
curl_easy_cleanup(t->curl);
if (t->tmp_filename)
free(t->tmp_filename);
}
return 0;
}
static void merge_files(transfert_t* transferts, int nb_threads,
char* out_filename)
{
int fd, fd_tmp, i;
struct stat s;
char* buffer;
rename(transferts[0].tmp_filename, out_filename);
fd = open(out_filename, O_WRONLY | O_APPEND);
for(i=1; i<nb_threads; i++)
{
fd_tmp = open(transferts[i].tmp_filename, O_RDONLY);
stat(transferts[i].tmp_filename, &s);
buffer = malloc(s.st_size);
read(fd_tmp, buffer, s.st_size);
write(fd, buffer, s.st_size);
free(buffer);
close(fd_tmp);
unlink(transferts[i].tmp_filename);
}
close(fd);
}
static int get_file_info(char* url, char* user_agent,
char** filename, unsigned* size,
unsigned quiet)
{
double length;
CURLcode res;
int ret = 0;
unsigned response_code;
char* real_name = NULL;
CURL* curl = curl_easy_init();
if(!curl)
return -1;
curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent);
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
if (!quiet)
curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
ret = -1;
goto end;
}
curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &response_code);
if (response_code != 200)
{
ret = -1;
goto end;
}
curl_easy_getinfo( curl,
CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length );
*size = length;
if (*filename == NULL)
{
curl_easy_getinfo( curl, CURLINFO_EFFECTIVE_URL, &real_name );
if (real_name)
{
*filename = strdup(basename(real_name));
if (!strcmp(*filename, "."))
{
free(*filename);
*filename = strdup(DEFAULT_OUT_FILENAME);
printf("Filename not found, output to %s\n", DEFAULT_OUT_FILENAME);
}
}
}
end:
curl_easy_cleanup(curl);
return ret;
}
static void find_free_file(char** filename)
{
struct stat s;
int cur_index;
char* new_filename;
unsigned new_size;
if (stat(*filename, &s)) return ;
new_size = strlen(*filename) + 1 + 2 + 1;
new_filename = malloc(new_size);
for (cur_index = 1; cur_index < 100; cur_index++)
{
snprintf(new_filename, new_size, "%s.%d",
*filename, cur_index);
if (stat(new_filename, &s))
{
free(*filename);
*filename = new_filename;
return ;
}
}
free(*filename);
*filename = NULL;
}
static void usage(char* program_name)
{
printf("%s: Parallel HTTP file download\n", program_name);
printf("usage: %s [-n nb_threads] [-l speed_limit] [-o out_filename] [-u user_agent] [-m max_chunk_size[kKmMgG]] [-q] [-h] url\n",
program_name);
printf("\t-n : Specify number of threads (default : %d)\n", DEFAULT_NB_THREADS);
printf("\t-l : Download speed limit for all threads (not per thread)\n");
printf("\t-o : Out filename, default is retrieved by GET request or '%s' if not found\n",
DEFAULT_OUT_FILENAME);
printf("\t-u : User agent, default is '%s'\n", DEFAULT_USER_AGENT);
printf("\t-m : Max chunk size in bytes\n");
printf("\t-q : Quiet mode\n");
printf("\t-h : Display help\n");
}
int main(int argc, char** argv)
{
pthread_t display_thread = 0;
unsigned nb_threads = DEFAULT_NB_THREADS;
char* user_agent = strdup(DEFAULT_USER_AGENT), *endptr;
stats_t* stats = NULL;
int ret = -1, i;
stats_params_t stats_params;
transfert_t* transferts = NULL;
unsigned total_size, thread_size, multiplier=1;
double displayed_size;
unsigned start, end, max_speed = 0, quiet = 0, max_chunk_size = 0;
char* out_filename = NULL, *url = NULL, c;
char* suffix = "B";
void* res;
int opt;
struct timeval time_start, time_end;
struct tm * full_time;
double average_speed;
int continuous_mode=0;
int exists;
while ((opt = getopt(argc, argv, "hl:m:n:o:qu:")) != -1) {
switch (opt) {
case 'l':
max_speed = strtoul(optarg, &endptr, 0);
if (*endptr)
{
usage(argv[0]);
return 1;
}
break;
case 'm':
if (strlen(optarg) > 1)
{
c = optarg[strlen(optarg)-1];
if (c < '0' || c > '9')
{
switch(c)
{
case 'g':
case 'G':
multiplier *= 1024;
case 'm':
case 'M':
multiplier *= 1024;
case 'k':
case 'K':
multiplier *= 1024;
optarg[strlen(optarg)-1] = 0;
break;
default:
usage(argv[0]);
return 1;
}
}
}
max_chunk_size = strtoul(optarg, &endptr, 0);
if (*endptr)
{
usage(argv[0]);
return 1;
}
max_chunk_size *= multiplier;
break;
case 'n':
nb_threads = strtoul(optarg, &endptr, 0);
if (*endptr)
{
usage(argv[0]);
return 1;
}
if (nb_threads == 0)
nb_threads = DEFAULT_NB_THREADS;
else if (nb_threads > MAX_NB_THREADS)
{
printf("Max numberb of threads is %d\n", MAX_NB_THREADS);
nb_threads = MAX_NB_THREADS;
}
break;
case 'o':
out_filename = strdup(optarg);
break;
case 'q':
quiet = 1;
break;
case 'u':
user_agent = strdup(optarg);
break;
default:
case 'h':
usage(argv[0]);
return 0;
}
}
if (optind != argc-1)
{
usage(argv[0]);
return 0;
}
else
url = argv[optind];
if (get_file_info(url, user_agent,
&out_filename, &total_size,
quiet))
return -1;
if (!out_filename)
out_filename = strdup(DEFAULT_OUT_FILENAME);
find_free_file(&out_filename);
if (out_filename == NULL)
{
printf("Unable to find free file to write to !\n");
goto end;
}
else
{
displayed_size = (double)total_size;
suffix = "B";
if (displayed_size > (1024))
{
displayed_size /= 1024.0;
suffix = "kB";
}
if (displayed_size > (1024))
{
displayed_size /= 1024.0;
suffix = "MB";
}
if (displayed_size > (1024))
{
displayed_size /= 1024.0;
suffix = "GB";
}
printf("Save in '%s' (%.2f%s)\n\n", out_filename, displayed_size, suffix);
}
stats = malloc(sizeof(*stats)*nb_threads);
memset(stats, 0, sizeof(*stats)*nb_threads);
transferts = malloc(sizeof(*transferts)*(nb_threads+1));
memset(transferts, 0, sizeof(*transferts)*(nb_threads+1));
thread_size = total_size/nb_threads;
max_speed /= nb_threads;
for(i=0; i<nb_threads; i++)
{
transferts[i].stats = stats;
start = thread_size*i;
if (i < (nb_threads-1))
end = thread_size*(i+1)-1;
else
end = total_size;
ret = configure_transfert(i, &transferts[i], url, out_filename,
start, end, max_speed, user_agent,
max_chunk_size, &exists);
if (ret)
goto end;
transferts[i].stats[i].dltotal = transferts[i].already_downloaded +
(end - start);
transferts[i].stats[i].dlnow = transferts[i].already_downloaded;
transferts[i].stats[i].dllast = 0;
// First set continuous mode
if (i == 0)
{
continuous_mode = exists;
}
else
{
// Check valid continuous mode or not
if (exists ^ continuous_mode)
{
printf("Error : you already started to download this file with a different number of thread, please clear temporary files or restart with the same number of threads\n");
goto end;
}
}
}
// Check for last temporary file
configure_transfert(i, &transferts[i], url, out_filename,
start, end, max_speed, user_agent, max_chunk_size,
&exists);
unlink(transferts[i].tmp_filename);
if (exists)
{
printf("Error : you already started to download this file with a different number of thread, please clear temporary files or restart with the same number of threads\n");
free_transfert(&transferts[i]);
goto end;
}
stats_params.end = 0;
stats_params.nb_threads = nb_threads;
stats_params.stats = stats;
if (!quiet)
{
pthread_create(&display_thread, NULL,
(void*(*)(void*))display_stats,
(void*) &stats_params);
}
gettimeofday(&time_start, NULL);
for(i=0; i<nb_threads; i++)
{
pthread_create(&transferts[i].thread, NULL,
(void*(*)(void*))do_transfert,
(void*) &transferts[i]);
}
for(i=0; i<nb_threads; i++)
{
pthread_join(transferts[i].thread, &res);
}
gettimeofday(&time_end, NULL);
merge_files(transferts, nb_threads, out_filename);
if (!quiet)
{
full_time = localtime((const time_t*)&time_end.tv_sec);
average_speed = (double)total_size / (double)(time_end.tv_sec - time_start.tv_sec);
if (average_speed < (1024*1024))
{
average_speed /= 1024;
suffix = "kB";
}
else
{
average_speed /= 1024*1024;
suffix = "MB";
}
printf("\n\n%04d-%02d-%02d %02d:%02d:%02d (%.2f%s/s) '%s' saved\n",
full_time->tm_year+1900, full_time->tm_mon, full_time->tm_mday,
full_time->tm_hour, full_time->tm_min, full_time->tm_sec,
average_speed, suffix, out_filename);
}
ret = 0;
end:
if (stats)
free(stats);
free(user_agent);
free(out_filename);
stats_params.end = 1;
for(i=0; i<nb_threads; i++)
free_transfert(&transferts[i]);
if (!quiet)
pthread_join(display_thread, &res);
if (transferts)
free(transferts);
if (!quiet)
printf("\n");
return ret;
}