unix prepend line to file without buffer/copy

Question

I want this, the sed solution looks good, but when I tried it seems to create a huge temp file.

unix command to prepend text to a file

I have big sql file that is like 28 gigs, I dont have much space on file system and just want to add one line to the front of the file. How can I do this without using up more file space?

Answer 1

Unfortunately, on every OS and file system I've ever seen, prepending can't generally be done in place like appending can. One might argue that file systems could do it efficiently, if the amount of data was some multiple of the underlying file system block size, but since that would not generally be the case, I don't know of any that have actually implemented such functionality. So, probably, the only way to do it is through a temporary file or copy. You could, however, use compression to somewhat alleviate the space crunch, but that would require some prework, and may not ultimately be suitable. Something along these lines:

1) gzip original_file.sql    # or bzip2 or whatever
2) create new_file.sql with data to be prepended
3) (cat new_file.sql; zcat original_file.sql.gz) | gzip > updated_file.sql.gz
4) zcat updated_file.sql.gz | less  # inspect the top of the file to make sure it looks right
5) rm original_file.sql.gz new_file.sql
6) gunzip updated_file.sql.gz # if necessary to have uncompressed text available - if not, just leave it compressed

Answer 2

Compile with gcc:

#include <stdio.h>
#include <string.h>
#include <malloc.h>

// Prepend size must be less than this value
#define bufSize 1024000

int main( int argc, char **argv )
{
    FILE *fil;
    unsigned char *smallBuf, *mainBuf;
    size_t sReadSize, mReadSize;
    long readPos = 0, writePos = 0;
    int appendSize;

    if( argc != 3 )
    {
        printf( "Usage: %s, <prepend_line> <file>\n", argv[0] );
        return 1;
    }

    sReadSize = appendSize = strlen( argv[1] ) + 1;

    smallBuf = (unsigned char *) malloc( appendSize );
    mainBuf = (unsigned char *) malloc( bufSize );
    if( !smallBuf || !mainBuf )
    {
        printf( "No memory\n" );
        return 1;
    }

    memcpy( smallBuf, argv[1], appendSize );
    smallBuf[ appendSize - 1 ] = '\n';

    fil = fopen( argv[2], "rb+" );
    if( !fil )
    {
        printf( "Cannot open file\n" );
        return 1;
    }

    while( 1 )
    {
        fseek( fil, readPos, 0 );
        readPos += mReadSize = fread( mainBuf, 1, bufSize, fil );

        fseek( fil, writePos, 0 );
        writePos += fwrite( smallBuf, 1, sReadSize, fil );

        if( mReadSize < bufSize )
        {
            if( mReadSize > 0 )
                fwrite( mainBuf, 1, mReadSize, fil );
            break;
        }

        fseek( fil, readPos, 0 );
        readPos += sReadSize = fread( smallBuf, 1, appendSize, fil );

        fseek( fil, writePos, 0 );
        writePos += fwrite( mainBuf, 1, mReadSize, fil );

        if( sReadSize < appendSize )
        {
            if( sReadSize > 0 )
                fwrite( smallBuf, 1, sReadSize, fil );
            break;
        }
    }

    fclose( fil );
    return 0;
}

Answer 3

You can use perl for this:

perl -i -n -e 'print "xxx\n$_" if $.==1;print if $.!=1' your_file