blob: 1310993bd54acaddb5fbe33fe38b97e4058b63e3 [file] [log] [blame]
Lucas Eckels9bd90e62012-08-06 15:07:02 -07001/*****************************************************************************
2 *
3 * This example source code introduces a c library buffered I/O interface to
4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5 * rewind(). Supported functions have identical prototypes to their normal c
6 * lib namesakes and are preceaded by url_ .
7 *
8 * Using this code you can replace your program's fopen() with url_fopen()
9 * and fread() with url_fread() and it become possible to read remote streams
10 * instead of (only) local files. Local files (ie those that can be directly
11 * fopened) will drop back to using the underlying clib implementations
12 *
13 * See the main() function at the bottom that shows an app that retrives from a
14 * specified url using fgets() and fread() and saves as two output files.
15 *
16 * Copyright (c) 2003 Simtec Electronics
17 *
18 * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19 * reference to original curl example code
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 * 1. Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 * 3. The name of the author may not be used to endorse or promote products
30 * derived from this software without specific prior written permission.
31 *
32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * This example requires libcurl 7.9.7 or later.
44 */
45
46#include <stdio.h>
47#include <string.h>
48#ifndef WIN32
49# include <sys/time.h>
50#endif
51#include <stdlib.h>
52#include <errno.h>
53
54#include <curl/curl.h>
55
56enum fcurl_type_e { CFTYPE_NONE=0, CFTYPE_FILE=1, CFTYPE_CURL=2 };
57
58struct fcurl_data
59{
60 enum fcurl_type_e type; /* type of handle */
61 union {
62 CURL *curl;
63 FILE *file;
64 } handle; /* handle */
65
66 char *buffer; /* buffer to store cached data*/
67 int buffer_len; /* currently allocated buffers length */
68 int buffer_pos; /* end of data in buffer*/
69 int still_running; /* Is background url fetch still in progress */
70};
71
72typedef struct fcurl_data URL_FILE;
73
74/* exported functions */
75URL_FILE *url_fopen(const char *url,const char *operation);
76int url_fclose(URL_FILE *file);
77int url_feof(URL_FILE *file);
78size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
79char * url_fgets(char *ptr, int size, URL_FILE *file);
80void url_rewind(URL_FILE *file);
81
82/* we use a global one for convenience */
83CURLM *multi_handle;
84
85/* curl calls this routine to get more data */
86static size_t
87write_callback(char *buffer,
88 size_t size,
89 size_t nitems,
90 void *userp)
91{
92 char *newbuff;
93 int rembuff;
94
95 URL_FILE *url = (URL_FILE *)userp;
96 size *= nitems;
97
98 rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
99
100 if(size > rembuff)
101 {
102 /* not enough space in buffer */
103 newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
104 if(newbuff==NULL)
105 {
106 fprintf(stderr,"callback buffer grow failed\n");
107 size=rembuff;
108 }
109 else
110 {
111 /* realloc suceeded increase buffer size*/
112 url->buffer_len+=size - rembuff;
113 url->buffer=newbuff;
114
115 /*printf("Callback buffer grown to %d bytes\n",url->buffer_len);*/
116 }
117 }
118
119 memcpy(&url->buffer[url->buffer_pos], buffer, size);
120 url->buffer_pos += size;
121
122 /*fprintf(stderr, "callback %d size bytes\n", size);*/
123
124 return size;
125}
126
127/* use to attempt to fill the read buffer up to requested number of bytes */
128static int
129fill_buffer(URL_FILE *file,int want,int waittime)
130{
131 fd_set fdread;
132 fd_set fdwrite;
133 fd_set fdexcep;
134 struct timeval timeout;
135 int rc;
136
137 /* only attempt to fill buffer if transactions still running and buffer
138 * doesnt exceed required size already
139 */
140 if((!file->still_running) || (file->buffer_pos > want))
141 return 0;
142
143 /* attempt to fill buffer */
144 do
145 {
146 int maxfd = -1;
147 long curl_timeo = -1;
148
149 FD_ZERO(&fdread);
150 FD_ZERO(&fdwrite);
151 FD_ZERO(&fdexcep);
152
153 /* set a suitable timeout to fail on */
154 timeout.tv_sec = 60; /* 1 minute */
155 timeout.tv_usec = 0;
156
157 curl_multi_timeout(multi_handle, &curl_timeo);
158 if(curl_timeo >= 0) {
159 timeout.tv_sec = curl_timeo / 1000;
160 if(timeout.tv_sec > 1)
161 timeout.tv_sec = 1;
162 else
163 timeout.tv_usec = (curl_timeo % 1000) * 1000;
164 }
165
166 /* get file descriptors from the transfers */
167 curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
168
169 /* In a real-world program you OF COURSE check the return code of the
170 function calls. On success, the value of maxfd is guaranteed to be
171 greater or equal than -1. We call select(maxfd + 1, ...), specially
172 in case of (maxfd == -1), we call select(0, ...), which is basically
173 equal to sleep. */
174
175 rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
176
177 switch(rc) {
178 case -1:
179 /* select error */
180 break;
181
182 case 0:
183 break;
184
185 default:
186 /* timeout or readable/writable sockets */
187 curl_multi_perform(multi_handle, &file->still_running);
188 break;
189 }
190 } while(file->still_running && (file->buffer_pos < want));
191 return 1;
192}
193
194/* use to remove want bytes from the front of a files buffer */
195static int
196use_buffer(URL_FILE *file,int want)
197{
198 /* sort out buffer */
199 if((file->buffer_pos - want) <=0)
200 {
201 /* ditch buffer - write will recreate */
202 if(file->buffer)
203 free(file->buffer);
204
205 file->buffer=NULL;
206 file->buffer_pos=0;
207 file->buffer_len=0;
208 }
209 else
210 {
211 /* move rest down make it available for later */
212 memmove(file->buffer,
213 &file->buffer[want],
214 (file->buffer_pos - want));
215
216 file->buffer_pos -= want;
217 }
218 return 0;
219}
220
221
222
223URL_FILE *
224url_fopen(const char *url,const char *operation)
225{
226 /* this code could check for URLs or types in the 'url' and
227 basicly use the real fopen() for standard files */
228
229 URL_FILE *file;
230 (void)operation;
231
232 file = malloc(sizeof(URL_FILE));
233 if(!file)
234 return NULL;
235
236 memset(file, 0, sizeof(URL_FILE));
237
238 if((file->handle.file=fopen(url,operation)))
239 {
240 file->type = CFTYPE_FILE; /* marked as URL */
241 }
242 else
243 {
244 file->type = CFTYPE_CURL; /* marked as URL */
245 file->handle.curl = curl_easy_init();
246
247 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
248 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
249 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
250 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
251
252 if(!multi_handle)
253 multi_handle = curl_multi_init();
254
255 curl_multi_add_handle(multi_handle, file->handle.curl);
256
257 /* lets start the fetch */
258 curl_multi_perform(multi_handle, &file->still_running);
259
260 if((file->buffer_pos == 0) && (!file->still_running))
261 {
262 /* if still_running is 0 now, we should return NULL */
263
264 /* make sure the easy handle is not in the multi handle anymore */
265 curl_multi_remove_handle(multi_handle, file->handle.curl);
266
267 /* cleanup */
268 curl_easy_cleanup(file->handle.curl);
269
270 free(file);
271
272 file = NULL;
273 }
274 }
275 return file;
276}
277
278int
279url_fclose(URL_FILE *file)
280{
281 int ret=0;/* default is good return */
282
283 switch(file->type)
284 {
285 case CFTYPE_FILE:
286 ret=fclose(file->handle.file); /* passthrough */
287 break;
288
289 case CFTYPE_CURL:
290 /* make sure the easy handle is not in the multi handle anymore */
291 curl_multi_remove_handle(multi_handle, file->handle.curl);
292
293 /* cleanup */
294 curl_easy_cleanup(file->handle.curl);
295 break;
296
297 default: /* unknown or supported type - oh dear */
298 ret=EOF;
299 errno=EBADF;
300 break;
301
302 }
303
304 if(file->buffer)
305 free(file->buffer);/* free any allocated buffer space */
306
307 free(file);
308
309 return ret;
310}
311
312int
313url_feof(URL_FILE *file)
314{
315 int ret=0;
316
317 switch(file->type)
318 {
319 case CFTYPE_FILE:
320 ret=feof(file->handle.file);
321 break;
322
323 case CFTYPE_CURL:
324 if((file->buffer_pos == 0) && (!file->still_running))
325 ret = 1;
326 break;
327 default: /* unknown or supported type - oh dear */
328 ret=-1;
329 errno=EBADF;
330 break;
331 }
332 return ret;
333}
334
335size_t
336url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
337{
338 size_t want;
339
340 switch(file->type)
341 {
342 case CFTYPE_FILE:
343 want=fread(ptr,size,nmemb,file->handle.file);
344 break;
345
346 case CFTYPE_CURL:
347 want = nmemb * size;
348
349 fill_buffer(file,want,1);
350
351 /* check if theres data in the buffer - if not fill_buffer()
352 * either errored or EOF */
353 if(!file->buffer_pos)
354 return 0;
355
356 /* ensure only available data is considered */
357 if(file->buffer_pos < want)
358 want = file->buffer_pos;
359
360 /* xfer data to caller */
361 memcpy(ptr, file->buffer, want);
362
363 use_buffer(file,want);
364
365 want = want / size; /* number of items - nb correct op - checked
366 * with glibc code*/
367
368 /*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/
369 break;
370
371 default: /* unknown or supported type - oh dear */
372 want=0;
373 errno=EBADF;
374 break;
375
376 }
377 return want;
378}
379
380char *
381url_fgets(char *ptr, int size, URL_FILE *file)
382{
383 int want = size - 1;/* always need to leave room for zero termination */
384 int loop;
385
386 switch(file->type)
387 {
388 case CFTYPE_FILE:
389 ptr = fgets(ptr,size,file->handle.file);
390 break;
391
392 case CFTYPE_CURL:
393 fill_buffer(file,want,1);
394
395 /* check if theres data in the buffer - if not fill either errored or
396 * EOF */
397 if(!file->buffer_pos)
398 return NULL;
399
400 /* ensure only available data is considered */
401 if(file->buffer_pos < want)
402 want = file->buffer_pos;
403
404 /*buffer contains data */
405 /* look for newline or eof */
406 for(loop=0;loop < want;loop++)
407 {
408 if(file->buffer[loop] == '\n')
409 {
410 want=loop+1;/* include newline */
411 break;
412 }
413 }
414
415 /* xfer data to caller */
416 memcpy(ptr, file->buffer, want);
417 ptr[want]=0;/* allways null terminate */
418
419 use_buffer(file,want);
420
421 /*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/
422 break;
423
424 default: /* unknown or supported type - oh dear */
425 ptr=NULL;
426 errno=EBADF;
427 break;
428 }
429
430 return ptr;/*success */
431}
432
433void
434url_rewind(URL_FILE *file)
435{
436 switch(file->type)
437 {
438 case CFTYPE_FILE:
439 rewind(file->handle.file); /* passthrough */
440 break;
441
442 case CFTYPE_CURL:
443 /* halt transaction */
444 curl_multi_remove_handle(multi_handle, file->handle.curl);
445
446 /* restart */
447 curl_multi_add_handle(multi_handle, file->handle.curl);
448
449 /* ditch buffer - write will recreate - resets stream pos*/
450 if(file->buffer)
451 free(file->buffer);
452
453 file->buffer=NULL;
454 file->buffer_pos=0;
455 file->buffer_len=0;
456
457 break;
458
459 default: /* unknown or supported type - oh dear */
460 break;
461
462 }
463
464}
465
466
467/* Small main program to retrive from a url using fgets and fread saving the
468 * output to two test files (note the fgets method will corrupt binary files if
469 * they contain 0 chars */
470int
471main(int argc, char *argv[])
472{
473 URL_FILE *handle;
474 FILE *outf;
475
476 int nread;
477 char buffer[256];
478 const char *url;
479
480 if(argc < 2)
481 {
482 url="http://192.168.7.3/testfile";/* default to testurl */
483 }
484 else
485 {
486 url=argv[1];/* use passed url */
487 }
488
489 /* copy from url line by line with fgets */
490 outf=fopen("fgets.test","w+");
491 if(!outf)
492 {
493 perror("couldn't open fgets output file\n");
494 return 1;
495 }
496
497 handle = url_fopen(url, "r");
498 if(!handle)
499 {
500 printf("couldn't url_fopen() %s\n", url);
501 fclose(outf);
502 return 2;
503 }
504
505 while(!url_feof(handle))
506 {
507 url_fgets(buffer,sizeof(buffer),handle);
508 fwrite(buffer,1,strlen(buffer),outf);
509 }
510
511 url_fclose(handle);
512
513 fclose(outf);
514
515
516 /* Copy from url with fread */
517 outf=fopen("fread.test","w+");
518 if(!outf)
519 {
520 perror("couldn't open fread output file\n");
521 return 1;
522 }
523
524 handle = url_fopen("testfile", "r");
525 if(!handle) {
526 printf("couldn't url_fopen() testfile\n");
527 fclose(outf);
528 return 2;
529 }
530
531 do {
532 nread = url_fread(buffer, 1,sizeof(buffer), handle);
533 fwrite(buffer,1,nread,outf);
534 } while(nread);
535
536 url_fclose(handle);
537
538 fclose(outf);
539
540
541 /* Test rewind */
542 outf=fopen("rewind.test","w+");
543 if(!outf)
544 {
545 perror("couldn't open fread output file\n");
546 return 1;
547 }
548
549 handle = url_fopen("testfile", "r");
550 if(!handle) {
551 printf("couldn't url_fopen() testfile\n");
552 fclose(outf);
553 return 2;
554 }
555
556 nread = url_fread(buffer, 1,sizeof(buffer), handle);
557 fwrite(buffer,1,nread,outf);
558 url_rewind(handle);
559
560 buffer[0]='\n';
561 fwrite(buffer,1,1,outf);
562
563 nread = url_fread(buffer, 1,sizeof(buffer), handle);
564 fwrite(buffer,1,nread,outf);
565
566
567 url_fclose(handle);
568
569 fclose(outf);
570
571
572 return 0;/* all done */
573}