/* Copyright 2014-2016 Grégory Soutadé This file is part of gget. gget is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. gget is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with gget. If not, see . */ #include #include #include #include #include #include #include #include #include #include #define DEFAULT_USER_AGENT "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0" #define DEFAULT_NB_THREADS 3 #define MAX_NB_THREADS 10 #define DEFAULT_OUT_FILENAME "gget.out" #ifdef WIN32 #include #define GNU_ERR "" typedef HANDLE pthread_t; typedef void pthread_attr_t; static int get_console_width() { RECT r; HWND console = GetConsoleWindow(); GetWindowRect(console, &r); return r.right-r.left; } static int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg) { *thread = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)start_routine, arg, 0, NULL); return 0; } static int pthread_join(pthread_t thread, void **retval) { WaitForSingleObject(thread, 0); GetExitCodeThread(thread,(LPDWORD)retval); CloseHandle(thread); return 0; } #else #include #include #define GNU_ERR " (%m)" static int get_console_width() { struct winsize w; ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); return w.ws_col; } #endif typedef struct { curl_off_t dltotal; curl_off_t dlnow; curl_off_t dllast; unsigned speed; } stats_t ; typedef struct { pthread_t thread; char* tmp_filename; // CURL *curl; char* url; char* user_agent; int fd; int id; unsigned already_downloaded; unsigned start; unsigned end; unsigned max_chunk_size; curl_off_t max_speed; stats_t* stats; } transfert_t; typedef struct { unsigned nb_threads; stats_t* stats; unsigned end; } stats_params_t; static void* display_stats(stats_params_t* p) { unsigned speed, percent, time_left, max_time_left; unsigned total_percent, hours, minutes, seconds; char* suffix; stats_t* cur; int i, nb_chars_printed; while (!p->end) { // If the window has been resized max_time_left = 0; suffix = "B"; total_percent = 0; nb_chars_printed = 0; printf("\r"); for(i=0; inb_threads; i++) { cur = &p->stats[i]; speed = cur->speed; if (cur->dltotal) percent = (cur->dlnow*100)/cur->dltotal; else percent = 0; if (speed) time_left = (cur->dltotal-cur->dlnow)/speed; else time_left = 0; if (speed < (1024*1024)) { speed /= 1024; suffix = "kB"; } else { speed /= 1024*1024; suffix = "MB"; } nb_chars_printed += printf("T%d: %02u%% %u%s/s ", i, percent, speed, suffix); if (time_left > max_time_left) max_time_left = time_left; total_percent += percent/p->nb_threads; } nb_chars_printed += printf(" Total: %u%% ", total_percent); if (max_time_left < 60) { nb_chars_printed += printf("eta %us", max_time_left); } else { seconds = max_time_left % 60; max_time_left /= 60; if (max_time_left < 60) { minutes = max_time_left; nb_chars_printed += printf("eta %um %us", minutes, seconds); } else { minutes = max_time_left % 60; hours = max_time_left / 60; nb_chars_printed += printf("eta %uh %um %us", hours, minutes, seconds); } } for(i=nb_chars_printed; ifd, contents, size*nmemb); if (ret < 0) printf("Error write" GNU_ERR "\n"); return ret; } static int progress_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { transfert_t* t = (transfert_t*)clientp; double speed_d; curl_easy_getinfo(t->curl, CURLINFO_SPEED_DOWNLOAD, &speed_d); t->stats[t->id].dlnow += (dlnow - t->stats[t->id].dllast); t->stats[t->id].dllast = dlnow; t->stats[t->id].speed = speed_d; return 0; } void* do_transfert(transfert_t* t) { CURLcode res; char range[64]; unsigned start, end, chunk_size; curl_easy_setopt(t->curl, CURLOPT_USERAGENT, t->user_agent); curl_easy_setopt(t->curl, CURLOPT_URL, t->url); curl_easy_setopt(t->curl, CURLOPT_FOLLOWLOCATION, 1L); if (t->max_speed) curl_easy_setopt(t->curl, CURLOPT_MAX_RECV_SPEED_LARGE, t->max_speed); /* send all data to this function */ curl_easy_setopt(t->curl, CURLOPT_WRITEFUNCTION, write_cb); /* we pass our 'chunk' struct to the callback function */ curl_easy_setopt(t->curl, CURLOPT_WRITEDATA, (void*)t); curl_easy_setopt(t->curl, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(t->curl, CURLOPT_XFERINFOFUNCTION, progress_cb); curl_easy_setopt(t->curl, CURLOPT_XFERINFODATA, t); start = t->start; if (t->max_chunk_size && (t->end - t->start) > t->max_chunk_size) chunk_size = t->max_chunk_size; else chunk_size = t->end - t->start; end = start + chunk_size; while (start < t->end) { snprintf(range, sizeof(range), "%u-%u", start, end); curl_easy_setopt(t->curl, CURLOPT_RANGE, range); /* Perform the request, res will get the return code */ res = curl_easy_perform(t->curl); /* Check for errors */ if(res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); break; } start += chunk_size + 1; if (t->max_chunk_size && (t->end - start) > t->max_chunk_size) chunk_size = t->max_chunk_size; else chunk_size = t->end - start; end = start + chunk_size; t->stats[t->id].dllast = 0; } return NULL; } static int configure_transfert(int id, transfert_t* t, char* url, char* filename, unsigned start, unsigned end, unsigned max_speed, char* user_agent, unsigned max_chunk_size, int* exists) { // filename + . + number + \0 unsigned filename_size = strlen(filename)+1+3+1; struct stat s; t->curl = curl_easy_init(); if(!t->curl) return -1; t->url = url; t->tmp_filename = malloc(filename_size); snprintf(t->tmp_filename, filename_size, "%s.%d", filename, id); t->id = id; t->start = start; t->end = end; t->max_speed = max_speed; t->user_agent = user_agent; t->max_chunk_size = max_chunk_size; if (stat(t->tmp_filename, &s)) { *exists = 0; t->fd = open(t->tmp_filename, O_WRONLY | O_CREAT, S_IRUSR|S_IWUSR); t->already_downloaded = 0; } else { *exists = 1; t->fd = open(t->tmp_filename, O_WRONLY | O_APPEND); t->start += s.st_size; t->already_downloaded = s.st_size; } if (!t->fd) { printf("Opening '%s' failed" GNU_ERR "\n", t->tmp_filename); return -1; } return 0; } static int free_transfert(transfert_t* t) { if (t) { if (t->curl) curl_easy_cleanup(t->curl); if (t->tmp_filename) free(t->tmp_filename); } return 0; } static void merge_files(transfert_t* transferts, int nb_threads, char* out_filename) { int fd, fd_tmp, i; struct stat s; char* buffer; rename(transferts[0].tmp_filename, out_filename); fd = open(out_filename, O_WRONLY | O_APPEND); for(i=1; i 1) { c = optarg[strlen(optarg)-1]; if (c < '0' || c > '9') { switch(c) { case 'g': case 'G': multiplier *= 1024; case 'm': case 'M': multiplier *= 1024; case 'k': case 'K': multiplier *= 1024; optarg[strlen(optarg)-1] = 0; break; default: usage(argv[0]); return 1; } } } max_chunk_size = strtoul(optarg, &endptr, 0); if (*endptr) { usage(argv[0]); return 1; } max_chunk_size *= multiplier; break; case 'n': nb_threads = strtoul(optarg, &endptr, 0); if (*endptr) { usage(argv[0]); return 1; } if (nb_threads == 0) nb_threads = DEFAULT_NB_THREADS; else if (nb_threads > MAX_NB_THREADS) { printf("Max numberb of threads is %d\n", MAX_NB_THREADS); nb_threads = MAX_NB_THREADS; } break; case 'o': out_filename = strdup(optarg); break; case 'q': quiet = 1; break; case 'u': user_agent = strdup(optarg); break; default: case 'h': usage(argv[0]); return 0; } } if (optind != argc-1) { usage(argv[0]); return 0; } else url = argv[optind]; if (get_file_info(url, user_agent, &out_filename, &total_size, quiet)) return -1; if (!out_filename) out_filename = strdup(DEFAULT_OUT_FILENAME); find_free_file(&out_filename); if (out_filename == NULL) { printf("Unable to find free file to write to !\n"); goto end; } else { displayed_size = (double)total_size; suffix = "B"; if (displayed_size > (1024)) { displayed_size /= 1024.0; suffix = "kB"; } if (displayed_size > (1024)) { displayed_size /= 1024.0; suffix = "MB"; } if (displayed_size > (1024)) { displayed_size /= 1024.0; suffix = "GB"; } printf("Save in '%s' (%.2f%s)\n\n", out_filename, displayed_size, suffix); } stats = malloc(sizeof(*stats)*nb_threads); memset(stats, 0, sizeof(*stats)*nb_threads); transferts = malloc(sizeof(*transferts)*(nb_threads+1)); memset(transferts, 0, sizeof(*transferts)*(nb_threads+1)); thread_size = total_size/nb_threads; max_speed /= nb_threads; for(i=0; itm_year+1900, full_time->tm_mon, full_time->tm_mday, full_time->tm_hour, full_time->tm_min, full_time->tm_sec, average_speed, suffix, out_filename); } ret = 0; end: if (stats) free(stats); free(user_agent); free(out_filename); stats_params.end = 1; for(i=0; i